From df630785b3190ad34cf44e56cabb76a055359477 Mon Sep 17 00:00:00 2001 From: Mohammad Date: Sun, 21 Jun 2026 22:33:41 +0300 Subject: [PATCH 01/27] Add queue ttft scorer. Signed-off-by: Mohammad --- .../modelselector/scorer/medianttft/README.md | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 pkg/framework/plugins/modelselector/scorer/medianttft/README.md diff --git a/pkg/framework/plugins/modelselector/scorer/medianttft/README.md b/pkg/framework/plugins/modelselector/scorer/medianttft/README.md new file mode 100644 index 00000000..1ab47d0c --- /dev/null +++ b/pkg/framework/plugins/modelselector/scorer/medianttft/README.md @@ -0,0 +1,76 @@ +# Median-TTFT Scorer + +Routes each request to the model with the lowest predicted TTFT under current load. + +## Equations + +Every TTFT decomposes as `TTFT = prefill_time + queue_wait`. + +**P10Low** — queue-free service floor (10th percentile, inflight_at_dispatch ≤ 2): +``` +P10Low ≈ prefill_time +``` +High-inflight observations are excluded, so P10Low is immune to burst flooding. +A 10-min window keeps the estimate alive during sustained overload. + +**Capacity** — updated only when `P50/P10Low ∈ [1.5, 3.0]` (balanced zone): +``` +capacity = inflightAtP50 × P10Low / (P50 − P10Low) +``` +Derived from `P50 - P10Low = P10Low * inflight/capacity`. where `P50 - P10Low` is estimating the waiting time. +`inflightAtP50` is the `inflight_at_dispatch` of the observation whose TTFT landed at +P50. Below 1.5 the denominator is too noisy; above 3.0, P50 is contaminated by flooding; capacity is frozen in both cases. + +**Scorer:** +``` +loadRatio = inflight / capacity +effectiveTTFT = P10Low × (1 + (loadRatio − 1)) when loadRatio > 1 + = P10Low otherwise +score = (maxTTFT − effectiveTTFT) / (maxTTFT − minTTFT) +``` +Within capacity: `effectiveTTFT = P10Low`, the queue-free baseline. +At 2× capacity: `effectiveTTFT = 2 × P10Low` — exactly what the queue model predicts. +Unobserved models score 1.0 (cold start) or 0.5 (idle alongside observed peers). + +## Why it should work physically + +Think of the model as C parallel slots, each finishing a request in P10Low seconds. +With N requests in the system, the server clears them at a rate of C every P10Low, so +draining the current backlog takes `(N / C) × P10Low`. That drain time is the queue +wait a new request sees: + +``` +TTFT = P10Low + (N / C) × P10Low = P10Low × (1 + N / C) +``` + +Rearranging for C at the observed operating point `(inflightAtP50, P50)`: + +``` +capacity C = inflightAtP50 × P10Low / (P50 − P10Low) +``` + +The same model gives the scorer's penalty directly: when `inflight > C`, the predicted +drain time exceeds P10Low, so `effectiveTTFT = P10Low × (inflight / C)`. + +## Possible Enhancements +### prompt-length awareness + +Prefill time scales roughly linearly with token count, but the scorer currently treats +all requests identically regardless of prompt size. The fix is to normalise TTFT by +character count in the tracker (`rate = TTFT / chars`) and scale at score time: +``` +effectiveTTFT = P10Low_rate × request_chars × max(1, inflight/capacity) +``` +This makes the scorer aware of the current request's weight without requiring a +model-specific tokeniser; character count (≈ tokens / 4) is a sufficient proxy. + +### Score-proportional picker + +`max-score-picker` sends 100% of traffic to the single winner, turning every small +score difference into a full traffic flip. This causes oscillation: the best model +overloads, all traffic switches to the other, the first model drains and wins again. +`score-proportional-picker` eliminates this by routing probabilistically: +``` +P(model i) ∝ score_i^(1/T) # T = temperature, default 1.0 +``` +At T = 1.0, a model scoring 0.8 vs 0.2 receives ≈ 80% vs 20% of requests. From 53299eea97d2751896c21883dd959f67fad115ed Mon Sep 17 00:00:00 2001 From: Mohammad Date: Mon, 22 Jun 2026 14:41:51 +0300 Subject: [PATCH 02/27] Add scorer implmenetation. Signed-off-by: Mohammad --- examples/medianttft-values.yaml | 48 +++ .../datalayer/ttftpercentile/plugin.go | 296 ++++++++++++++++++ .../datalayer/ttftpercentile/tracker.go | 102 ++++++ .../{medianttft => queuettft}/README.md | 0 .../modelselector/scorer/queuettft/plugin.go | 167 ++++++++++ 5 files changed, 613 insertions(+) create mode 100644 examples/medianttft-values.yaml create mode 100644 pkg/framework/plugins/datalayer/ttftpercentile/plugin.go create mode 100644 pkg/framework/plugins/datalayer/ttftpercentile/tracker.go rename pkg/framework/plugins/modelselector/scorer/{medianttft => queuettft}/README.md (100%) create mode 100644 pkg/framework/plugins/modelselector/scorer/queuettft/plugin.go diff --git a/examples/medianttft-values.yaml b/examples/medianttft-values.yaml new file mode 100644 index 00000000..78486b2e --- /dev/null +++ b/examples/medianttft-values.yaml @@ -0,0 +1,48 @@ +payloadProcessor: + listModels: + - facebook/opt-125m + - facebook/opt-350m + customConfig: + plugins: + - type: body-field-to-header + parameters: + fieldName: model + headerName: X-Gateway-Model-Name + - type: base-model-to-header + - type: model-selector + - type: median-ttft-scorer + parameters: + # effectiveTTFT = P10Low × (1 + inflightPenaltyWeight × (inflight/capacity − 1)) + # capacity = inflightAtP50 × P10Low / (P50 − P10Low) + # updated only when P50/P10Low ∈ [1.5, 3.0] (balanced zone) + # inflightAtP50 = inflight_at_dispatch of the observation whose TTFT is P50 + # (exact pairing — same request, no independent medians) + inflightPenaltyWeight: 1.0 + - type: max-score-picker + - type: ttft-percentile-extractor + parameters: + intervalDuration: 1s + windowSize: 1000 # buffer ≥ expected req/s × windowAge to keep window meaningful + windowAge: 5m # longer window → inflightAtP50 spans load transitions + minObservations: 2 + inflightEmaAlpha: 0.2 # EMA of inflight count (observability only) + lowInflightThreshold: 2 # observations at inflight≤N feed the stable P10Low tracker + lowLoadWindowAge: 10m # longer window: low-load obs become rare under heavy load + - type: model-config-datasource + parameters: + modelsPath: /config/models.json + profiles: + - name: default + plugins: + request: + - pluginRef: model-selector + - pluginRef: median-ttft-scorer + weight: 1.0 + - pluginRef: max-score-picker + - pluginRef: body-field-to-header + - pluginRef: base-model-to-header + datalayer: + extractors: + - pluginRef: ttft-percentile-extractor + datasources: + - pluginRef: model-config-datasource diff --git a/pkg/framework/plugins/datalayer/ttftpercentile/plugin.go b/pkg/framework/plugins/datalayer/ttftpercentile/plugin.go new file mode 100644 index 00000000..2e1bfca1 --- /dev/null +++ b/pkg/framework/plugins/datalayer/ttftpercentile/plugin.go @@ -0,0 +1,296 @@ +/* +Copyright 2026 The llm-d Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package ttftpercentile tracks per-model TTFT distributions and publishes +// P10Low, P50, and a capacity estimate for the median-ttft-scorer. +package ttftpercentile + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "sigs.k8s.io/controller-runtime/pkg/log" + + logutil "github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/logging" + "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/interface/datalayer" + dlsrc "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/interface/datalayer/datasource" + "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/interface/plugin" +) + +const ( + PluginType = "ttft-percentile-extractor" + AttributeKey = "ttft-percentile" + + balancedZoneLow = 1.5 + balancedZoneHigh = 3.0 + + defaultWindowSize = 1000 + defaultWindowAge = 5 * time.Minute + defaultMinObservations = 3 + defaultIntervalDuration = 5 * time.Second + defaultInflightEMAAlpha = 0.2 + defaultLowInflightThreshold = 2 + defaultLowLoadWindowAge = 10 * time.Minute +) + +var _ dlsrc.Extractor = &TTFTPercentileExtractor{} + +type TTFTPercentileExtractorConfig struct { + IntervalDuration string `json:"intervalDuration,omitempty"` + WindowSize int `json:"windowSize,omitempty"` + WindowAge string `json:"windowAge,omitempty"` + MinObservations int `json:"minObservations,omitempty"` + InflightEMAAlpha float64 `json:"inflightEmaAlpha,omitempty"` + LowInflightThreshold int `json:"lowInflightThreshold,omitempty"` + LowLoadWindowAge string `json:"lowLoadWindowAge,omitempty"` +} + +// TTFTPercentileMetrics is written to each model's attribute store every intervalDuration. +type TTFTPercentileMetrics struct { + Requests int64 + AvgInflight float64 + InflightAtP50 float64 // inflight_at_dispatch of the P50 observation + P10LowTTFT float64 // P10 from inflight≤lowInflightThreshold obs; queue-free floor + P10TTFT float64 // P10 from all obs + P50TTFT float64 // P50 from all obs + Capacity float64 // InflightAtP50 × P10Low / (P50 − P10Low); frozen outside balanced zone + LastObservedAt int64 +} + +func (m TTFTPercentileMetrics) Clone() datalayer.Cloneable { return m } + +type pendingEntry struct { + inflightAtDispatch int64 + dispatchedAt time.Time +} + +type modelPercentileState struct { + TTFTPercentileMetrics + intervalStart time.Time + tracker percentileTracker + lowLoadTracker percentileTracker + pending map[string]pendingEntry + avgInflightInit bool +} + +func (s *modelPercentileState) flush(now time.Time, windowAge, lowLoadWindowAge time.Duration, minObs int) { + p50, inflightAtP50, ok50 := s.tracker.quantileWithInflight(0.50, now, windowAge, minObs) + p10, ok10 := s.tracker.quantile(0.10, now, windowAge, minObs) + if ok50 { + s.P50TTFT, s.InflightAtP50, s.LastObservedAt = p50, inflightAtP50, now.UnixNano() + } + if ok10 { + s.P10TTFT = p10 + } + if p10low, ok := s.lowLoadTracker.quantile(0.10, now, lowLoadWindowAge, 1); ok { + s.P10LowTTFT = p10low + } + p10forCap := s.P10LowTTFT + if p10forCap == 0 { + p10forCap = s.P10TTFT + } + if ok50 && p10forCap > 0 { + if ratio := p50 / p10forCap; ratio >= balancedZoneLow && ratio <= balancedZoneHigh { + if denom := p50 - p10forCap; denom > 0 { + s.Capacity = inflightAtP50 * p10forCap / denom + } + } + } + s.intervalStart = now +} + +type TTFTPercentileExtractor struct { + typedName plugin.TypedName + ds datalayer.Datastore + state map[string]*modelPercentileState + windowSize int + windowAge time.Duration + minObservations int + intervalDuration time.Duration + inflightEMAAlpha float64 + lowInflightThreshold int + lowLoadWindowAge time.Duration +} + +func ExtractorFactory(name string, parameters json.RawMessage, h plugin.Handle) (plugin.Plugin, error) { + cfg := TTFTPercentileExtractorConfig{ + IntervalDuration: defaultIntervalDuration.String(), WindowSize: defaultWindowSize, + WindowAge: defaultWindowAge.String(), MinObservations: defaultMinObservations, + InflightEMAAlpha: defaultInflightEMAAlpha, LowInflightThreshold: defaultLowInflightThreshold, + LowLoadWindowAge: defaultLowLoadWindowAge.String(), + } + if len(parameters) > 0 { + if err := json.Unmarshal(parameters, &cfg); err != nil { + return nil, fmt.Errorf("failed to parse parameters for plugin %q: %w", name, err) + } + } + if cfg.WindowSize <= 0 { + return nil, fmt.Errorf("windowSize must be > 0 for plugin %q", name) + } + if cfg.MinObservations <= 0 { + return nil, fmt.Errorf("minObservations must be > 0 for plugin %q", name) + } + if cfg.InflightEMAAlpha <= 0 || cfg.InflightEMAAlpha > 1 { + return nil, fmt.Errorf("inflightEmaAlpha must be in (0,1] for plugin %q", name) + } + if cfg.LowInflightThreshold < 0 { + return nil, fmt.Errorf("lowInflightThreshold must be >= 0 for plugin %q", name) + } + interval, err := time.ParseDuration(cfg.IntervalDuration) + if err != nil { + return nil, fmt.Errorf("invalid intervalDuration %q for plugin %q: %w", cfg.IntervalDuration, name, err) + } + windowAge, err := time.ParseDuration(cfg.WindowAge) + if err != nil { + return nil, fmt.Errorf("invalid windowAge %q for plugin %q: %w", cfg.WindowAge, name, err) + } + lowLoadAge, err := time.ParseDuration(cfg.LowLoadWindowAge) + if err != nil { + return nil, fmt.Errorf("invalid lowLoadWindowAge %q for plugin %q: %w", cfg.LowLoadWindowAge, name, err) + } + return NewTTFTPercentileExtractor(h.Datastore()). + WithName(name).WithIntervalDuration(interval). + WithWindow(cfg.WindowSize, windowAge, cfg.MinObservations). + WithInflightEMAAlpha(cfg.InflightEMAAlpha). + WithLowLoad(cfg.LowInflightThreshold, lowLoadAge), nil +} + +func NewTTFTPercentileExtractor(ds datalayer.Datastore) *TTFTPercentileExtractor { + return &TTFTPercentileExtractor{ + typedName: plugin.TypedName{Type: PluginType, Name: PluginType}, + ds: ds, state: make(map[string]*modelPercentileState), + windowSize: defaultWindowSize, windowAge: defaultWindowAge, + minObservations: defaultMinObservations, intervalDuration: defaultIntervalDuration, + inflightEMAAlpha: defaultInflightEMAAlpha, lowInflightThreshold: defaultLowInflightThreshold, + lowLoadWindowAge: defaultLowLoadWindowAge, + } +} + +func (e *TTFTPercentileExtractor) TypedName() plugin.TypedName { return e.typedName } +func (e *TTFTPercentileExtractor) WithName(n string) *TTFTPercentileExtractor { + e.typedName.Name = n; return e +} +func (e *TTFTPercentileExtractor) WithIntervalDuration(d time.Duration) *TTFTPercentileExtractor { + e.intervalDuration = d; return e +} +func (e *TTFTPercentileExtractor) WithWindow(size int, age time.Duration, minObs int) *TTFTPercentileExtractor { + e.windowSize, e.windowAge, e.minObservations = size, age, minObs; return e +} +func (e *TTFTPercentileExtractor) WithInflightEMAAlpha(a float64) *TTFTPercentileExtractor { + e.inflightEMAAlpha = a; return e +} +func (e *TTFTPercentileExtractor) WithLowLoad(threshold int, age time.Duration) *TTFTPercentileExtractor { + e.lowInflightThreshold, e.lowLoadWindowAge = threshold, age; return e +} + +func (e *TTFTPercentileExtractor) Extract(ctx context.Context, events []dlsrc.Event) error { + debugLogger := log.FromContext(ctx).V(logutil.DEBUG) + now := time.Now() + updated := map[string]bool{} + + for _, ev := range events { + switch ev.Type { + case dlsrc.RequestEventType: + p, ok := ev.Payload.(dlsrc.RequestPayload) + if !ok { + continue + } + model, _ := p.Request.Body["model"].(string) + if model == "" { + continue + } + s := e.getOrCreate(model) + inflight := s.Requests + s.Requests++ + if reqID := p.Request.Headers["x-request-id"]; reqID != "" { + s.pending[reqID] = pendingEntry{inflightAtDispatch: inflight, dispatchedAt: now} + } + updated[model] = true + + case dlsrc.ResponseEventType: + p, ok := ev.Payload.(dlsrc.ResponsePayload) + if !ok { + continue + } + model, _ := p.Request.Body["model"].(string) + if model == "" { + continue + } + s := e.getOrCreate(model) + if s.Requests--; s.Requests < 0 { + s.Requests = 0 + } + reqID := p.Request.Headers["x-request-id"] + if p.TTFT > 0 { + ttft := p.TTFT.Seconds() + var inflightAtDispatch int64 + if entry, found := s.pending[reqID]; found && reqID != "" { + inflightAtDispatch = entry.inflightAtDispatch + if inflightAtDispatch <= int64(e.lowInflightThreshold) { + s.lowLoadTracker.add(ttft, inflightAtDispatch, now) + } + } + s.tracker.add(ttft, inflightAtDispatch, now) + } + if reqID != "" { + delete(s.pending, reqID) + } + if now.Sub(s.intervalStart) >= e.intervalDuration { + sample := float64(s.Requests) + if !s.avgInflightInit { + s.AvgInflight, s.avgInflightInit = sample, true + } else { + s.AvgInflight = e.inflightEMAAlpha*sample + (1-e.inflightEMAAlpha)*s.AvgInflight + } + s.flush(now, e.windowAge, e.lowLoadWindowAge, e.minObservations) + } + updated[model] = true + } + } + + for model := range updated { + s := e.state[model] + cutoff := now.Add(-e.windowAge) + for id, entry := range s.pending { + if entry.dispatchedAt.Before(cutoff) { + delete(s.pending, id) + } + } + m := s.TTFTPercentileMetrics + e.ds.GetOrCreateModel(model).GetAttributes().Put(AttributeKey, m) + debugLogger.Info("ttft-percentile wrote attribute", + "model", model, "Requests", m.Requests, "AvgInflight", m.AvgInflight, + "InflightAtP50", m.InflightAtP50, "P10Low_s", m.P10LowTTFT, + "P10_s", m.P10TTFT, "P50_s", m.P50TTFT, "Capacity", m.Capacity, + ) + } + return nil +} + +func (e *TTFTPercentileExtractor) getOrCreate(model string) *modelPercentileState { + if s, ok := e.state[model]; ok { + return s + } + s := &modelPercentileState{ + tracker: newSlidingWindowTracker(e.windowSize), + lowLoadTracker: newSlidingWindowTracker(e.windowSize), + pending: make(map[string]pendingEntry), + } + e.state[model] = s + return s +} diff --git a/pkg/framework/plugins/datalayer/ttftpercentile/tracker.go b/pkg/framework/plugins/datalayer/ttftpercentile/tracker.go new file mode 100644 index 00000000..cdeb46cd --- /dev/null +++ b/pkg/framework/plugins/datalayer/ttftpercentile/tracker.go @@ -0,0 +1,102 @@ +/* +Copyright 2026 The llm-d Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ttftpercentile + +import ( + "sort" + "time" +) + +type percentileTracker interface { + add(value float64, inflight int64, ts time.Time) + quantile(p float64, now time.Time, maxAge time.Duration, minCount int) (float64, bool) + // quantileWithInflight returns the p-th percentile TTFT and the inflight of the + // observation at that position — same request, self-consistent (TTFT, inflight) pair. + quantileWithInflight(p float64, now time.Time, maxAge time.Duration, minCount int) (float64, float64, bool) +} + +type observation struct { + value float64 + inflight int64 // inflight_at_dispatch + ts time.Time +} + +type slidingWindowTracker struct { + buf []observation + head int + n int +} + +func newSlidingWindowTracker(cap int) *slidingWindowTracker { + return &slidingWindowTracker{buf: make([]observation, cap)} +} + +func (t *slidingWindowTracker) add(value float64, inflight int64, ts time.Time) { + t.buf[t.head] = observation{value: value, inflight: inflight, ts: ts} + t.head = (t.head + 1) % len(t.buf) + if t.n < len(t.buf) { + t.n++ + } +} + +func (t *slidingWindowTracker) recentObs(now time.Time, maxAge time.Duration) []observation { + cutoff := now.Add(-maxAge) + cap := len(t.buf) + out := make([]observation, 0, t.n) + for i := 0; i < t.n; i++ { + obs := t.buf[(t.head-1-i+cap)%cap] + if !obs.ts.Before(cutoff) { + out = append(out, obs) + } + } + return out +} + +func (t *slidingWindowTracker) quantile(p float64, now time.Time, maxAge time.Duration, minCount int) (float64, bool) { + obs := t.recentObs(now, maxAge) + if len(obs) < minCount { + return 0, false + } + vals := make([]float64, len(obs)) + for i, o := range obs { + vals[i] = o.value + } + sort.Float64s(vals) + idx := p * float64(len(vals)-1) + lo := int(idx) + if lo+1 >= len(vals) { + return vals[lo], true + } + return vals[lo] + (idx-float64(lo))*(vals[lo+1]-vals[lo]), true +} + +func (t *slidingWindowTracker) quantileWithInflight(p float64, now time.Time, maxAge time.Duration, minCount int) (float64, float64, bool) { + obs := t.recentObs(now, maxAge) + if len(obs) < minCount { + return 0, 0, false + } + sort.Slice(obs, func(i, j int) bool { return obs[i].value < obs[j].value }) + idx := p * float64(len(obs)-1) + lo := int(idx) + var value float64 + if lo+1 >= len(obs) { + value = obs[lo].value + } else { + value = obs[lo].value + (idx-float64(lo))*(obs[lo+1].value-obs[lo].value) + } + return value, float64(obs[lo].inflight), true +} diff --git a/pkg/framework/plugins/modelselector/scorer/medianttft/README.md b/pkg/framework/plugins/modelselector/scorer/queuettft/README.md similarity index 100% rename from pkg/framework/plugins/modelselector/scorer/medianttft/README.md rename to pkg/framework/plugins/modelselector/scorer/queuettft/README.md diff --git a/pkg/framework/plugins/modelselector/scorer/queuettft/plugin.go b/pkg/framework/plugins/modelselector/scorer/queuettft/plugin.go new file mode 100644 index 00000000..bee478ec --- /dev/null +++ b/pkg/framework/plugins/modelselector/scorer/queuettft/plugin.go @@ -0,0 +1,167 @@ +/* +Copyright 2026 The llm-d Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package medianttft scores models by predicted TTFT under current load. +// Model P10Low × (1 + inflight/capacity) is used to predict queue wait, +// where capacity is estimated from paired (P50, inflightAtP50) observations. +package medianttft + +import ( + "context" + "encoding/json" + "fmt" + "math" + + "sigs.k8s.io/controller-runtime/pkg/log" + + logutil "github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/logging" + "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/interface/datalayer" + "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/interface/modelselector" + "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/interface/plugin" + "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/interface/requesthandling" + "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/plugins/datalayer/ttftpercentile" +) + +const ( + PluginType = "median-ttft-scorer" + defaultInflightPenaltyWeight = 1.0 +) + +var _ modelselector.Scorer = &MedianTTFTScorer{} + +type MedianTTFTScorerConfig struct { + // InflightPenaltyWeight scales the overload penalty: + // effectiveTTFT = P10Low × (1 + weight × (inflight/capacity − 1)) + // 0 disables the penalty; 1.0 (default) equals the queue model prediction. + InflightPenaltyWeight *float64 `json:"inflightPenaltyWeight,omitempty"` +} + +type MedianTTFTScorer struct { + typedName plugin.TypedName + inflightPenaltyWeight float64 +} + +func ScorerFactory(name string, parameters json.RawMessage, _ plugin.Handle) (plugin.Plugin, error) { + cfg := MedianTTFTScorerConfig{} + if len(parameters) > 0 { + if err := json.Unmarshal(parameters, &cfg); err != nil { + return nil, fmt.Errorf("failed to parse parameters for plugin %q: %w", name, err) + } + } + w := defaultInflightPenaltyWeight + if cfg.InflightPenaltyWeight != nil { + if *cfg.InflightPenaltyWeight < 0 { + return nil, fmt.Errorf("inflightPenaltyWeight must be >= 0 for plugin %q", name) + } + w = *cfg.InflightPenaltyWeight + } + return NewMedianTTFTScorer().WithName(name).WithInflightPenaltyWeight(w), nil +} + +func NewMedianTTFTScorer() *MedianTTFTScorer { + return &MedianTTFTScorer{ + typedName: plugin.TypedName{Type: PluginType, Name: PluginType}, + inflightPenaltyWeight: defaultInflightPenaltyWeight, + } +} + +func (s *MedianTTFTScorer) TypedName() plugin.TypedName { return s.typedName } +func (s *MedianTTFTScorer) WithName(name string) *MedianTTFTScorer { + s.typedName.Name = name; return s +} +func (s *MedianTTFTScorer) WithInflightPenaltyWeight(w float64) *MedianTTFTScorer { + s.inflightPenaltyWeight = w; return s +} + +// Score returns (maxTTFT − effectiveTTFT) / (maxTTFT − minTTFT) per model. +// Unobserved models score 1.0 (all unobserved) or 0.5 (some peers observed). +func (s *MedianTTFTScorer) Score(ctx context.Context, _ *plugin.CycleState, _ *requesthandling.InferenceRequest, models []datalayer.Model) map[datalayer.Model]float64 { + ttfts := make(map[datalayer.Model]float64, len(models)) + minTTFT, maxTTFT := math.MaxFloat64, 0.0 + allUnobserved := true + + for _, model := range models { + v := s.effectiveTTFT(ctx, model) + ttfts[model] = v + if v > 0 { + allUnobserved = false + if v > maxTTFT { + maxTTFT = v + } + if v < minTTFT { + minTTFT = v + } + } + } + + scores := make(map[datalayer.Model]float64, len(models)) + for _, model := range models { + v := ttfts[model] + switch { + case v == 0 && allUnobserved: + scores[model] = 1.0 + case v == 0: + scores[model] = 0.5 + case maxTTFT == minTTFT: + scores[model] = 1.0 + default: + scores[model] = (maxTTFT - v) / (maxTTFT - minTTFT) + } + } + + if dl := log.FromContext(ctx).V(logutil.DEBUG); dl.Enabled() { + for _, model := range models { + dl.Info("median-ttft score", "model", model.GetName(), "effectiveTTFT", ttfts[model], "score", scores[model]) + } + } + return scores +} + +func (s *MedianTTFTScorer) effectiveTTFT(ctx context.Context, model datalayer.Model) float64 { + val, ok := model.GetAttributes().Get(ttftpercentile.AttributeKey) + if !ok { + return 0 + } + m, ok := val.(ttftpercentile.TTFTPercentileMetrics) + if !ok { + return 0 + } + p10 := m.P10LowTTFT + if p10 == 0 { + p10 = m.P10TTFT + } + if p10 == 0 { + return 0 + } + + eff := p10 + var loadRatio float64 + if m.Capacity >= 1 { + loadRatio = float64(m.Requests) / m.Capacity + if loadRatio > 1 { + eff = p10 * (1 + s.inflightPenaltyWeight*(loadRatio-1)) + } + } + + if dl := log.FromContext(ctx).V(logutil.DEBUG); dl.Enabled() { + dl.Info("median-ttft effective", + "model", model.GetName(), "inflight", m.Requests, "capacity", m.Capacity, + "loadRatio", loadRatio, "P10Low_s", m.P10LowTTFT, "P10_s", m.P10TTFT, + "P50_s", m.P50TTFT, "effectiveTTFT", eff, + ) + } + return eff +} From 776117c7581d5122079db21df78f7439a8981279 Mon Sep 17 00:00:00 2001 From: Mohammad Date: Mon, 22 Jun 2026 18:51:09 +0300 Subject: [PATCH 03/27] Update the scorer. Signed-off-by: Mohammad --- cmd/runner/runner.go | 6 + examples/medianttft-values.yaml | 18 +-- examples/plot_capacity.py | 141 ++++++++++++++++++ .../datalayer/ttftpercentile/plugin.go | 116 +++++++------- .../datalayer/ttftpercentile/tracker.go | 60 +++++++- .../modelselector/scorer/queuettft/README.md | 81 +++++----- .../modelselector/scorer/queuettft/plugin.go | 59 ++------ pkg/handlers/server.go | 4 +- 8 files changed, 319 insertions(+), 166 deletions(-) create mode 100644 examples/plot_capacity.py diff --git a/cmd/runner/runner.go b/cmd/runner/runner.go index df46f69d..3b647a61 100644 --- a/cmd/runner/runner.go +++ b/cmd/runner/runner.go @@ -47,11 +47,14 @@ import ( "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/interface/datalayer/datasource" "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/interface/plugin" "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/interface/requesthandling" + modelconfigcollector "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/plugins/datalayer/modelconfigcollector" requestmetadata "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/plugins/datalayer/requestmetadata" + "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/plugins/datalayer/ttftpercentile" "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/plugins/modelselector/picker/maxscore" "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/plugins/modelselector/picker/random" "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/plugins/modelselector/picker/weightedrandom" inflightrequestsscorer "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/plugins/modelselector/scorer/inflightrequests" + medianttftscorer "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/plugins/modelselector/scorer/queuettft" "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/plugins/requesthandling/basemodelextractor" "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/plugins/requesthandling/bodyfieldtoheader" modelselectorplugin "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/plugins/requesthandling/modelselector" @@ -277,12 +280,15 @@ func (r *Runner) registerInTreePlugins() { plugin.Register(bodyfieldtoheader.BodyFieldToHeaderPluginType, bodyfieldtoheader.BodyFieldToHeaderPluginFactory) plugin.Register(basemodelextractor.BaseModelToHeaderPluginType, basemodelextractor.BaseModelToHeaderPluginFactory) plugin.Register(requestmetadata.PluginType, requestmetadata.ExtractorFactory) + plugin.Register(modelconfigcollector.PluginType, modelconfigcollector.DatasourceFactory) + plugin.Register(ttftpercentile.PluginType, ttftpercentile.ExtractorFactory) // register model selector plugins plugin.Register(random.RandomPickerType, random.RandomPickerFactory) plugin.Register(maxscore.MaxScorePickerType, maxscore.MaxScorePickerFactory) plugin.Register(weightedrandom.WeightedRandomPickerType, weightedrandom.WeightedRandomPickerFactory) plugin.Register(modelselectorplugin.ModelSelectorPluginType, modelselectorplugin.ModelSelectorPluginFactory) plugin.Register(inflightrequestsscorer.PluginType, inflightrequestsscorer.ScorerFactory) + plugin.Register(medianttftscorer.PluginType, medianttftscorer.ScorerFactory) } // registerHealthServer adds the Health gRPC server as a Runnable to the given manager. diff --git a/examples/medianttft-values.yaml b/examples/medianttft-values.yaml index 78486b2e..5ee2daa5 100644 --- a/examples/medianttft-values.yaml +++ b/examples/medianttft-values.yaml @@ -11,23 +11,17 @@ payloadProcessor: - type: base-model-to-header - type: model-selector - type: median-ttft-scorer - parameters: - # effectiveTTFT = P10Low × (1 + inflightPenaltyWeight × (inflight/capacity − 1)) - # capacity = inflightAtP50 × P10Low / (P50 − P10Low) - # updated only when P50/P10Low ∈ [1.5, 3.0] (balanced zone) - # inflightAtP50 = inflight_at_dispatch of the observation whose TTFT is P50 - # (exact pairing — same request, no independent medians) - inflightPenaltyWeight: 1.0 + # effectiveTTFT = P10Low + inflight x (P50 - P10Low) / inflightAtP50 + # line through (0, P10Low) and (inflightAtP50, P50), extrapolated linearly - type: max-score-picker - type: ttft-percentile-extractor parameters: intervalDuration: 1s - windowSize: 1000 # buffer ≥ expected req/s × windowAge to keep window meaningful - windowAge: 5m # longer window → inflightAtP50 spans load transitions - minObservations: 2 + windowSize: 5000 # ~200 KB per model; holds ~1h at 1 req/s or ~8 min at 10 req/s + windowAge: 1m # window for P50 and P10 (short = fresh, responsive) + minObservations: 3 inflightEmaAlpha: 0.2 # EMA of inflight count (observability only) - lowInflightThreshold: 2 # observations at inflight≤N feed the stable P10Low tracker - lowLoadWindowAge: 10m # longer window: low-load obs become rare under heavy load + lowLoadWindowAge: 1h # window for two-level P10Low (long = stable hardware floor) - type: model-config-datasource parameters: modelsPath: /config/models.json diff --git a/examples/plot_capacity.py b/examples/plot_capacity.py new file mode 100644 index 00000000..90b8e5e1 --- /dev/null +++ b/examples/plot_capacity.py @@ -0,0 +1,141 @@ +""" +Plot effective TTFT vs real TTFT from a ttft-percentile extractor log file. + +Produces one file with 4 panels: + 1. effectiveTTFT line + real TTFT scatter + 2. effectiveTTFT line + real TTFT binned median line + 3. Inflight requests + 4. P10Low and P50 TTFT lines + +Usage: python plot_capacity.py +""" + +import json +import statistics +import sys +from collections import defaultdict + +import matplotlib.pyplot as plt + +LOG_FILE = sys.argv[1] if len(sys.argv) > 1 else "logs/bench_median33" +BIN_SEC = 10 # bucket width for the median graph + +# --------------------------------------------------------------------------- +# Parse +# --------------------------------------------------------------------------- +flush_records = defaultdict(list) +obs_records = defaultdict(list) + +with open(LOG_FILE) as f: + for line in f: + line = line.strip() + if not line: + continue + try: + obj = json.loads(line) + except json.JSONDecodeError: + continue + msg = obj.get("msg", "") + model = obj.get("model", "") + if not model: + continue + if msg == "ttft-percentile wrote attribute": + flush_records[model].append({ + "ts": obj["ts"], + "inflight": obj.get("Requests", 0), + "p10low": obj.get("P10Low_s", 0), + "p50": obj.get("P50_s", 0), + "effective":obj.get("EffectiveTTFT_s", 0), + }) + elif msg == "ttft-observation": + obs_records[model].append({ + "ts": obj["ts"], + "ttft": obj.get("ttft_s", 0), + }) + +if not flush_records: + print("No 'ttft-percentile wrote attribute' lines found in", LOG_FILE) + sys.exit(1) + +# Normalise timestamps +t0 = min( + r["ts"] + for recs in list(flush_records.values()) + list(obs_records.values()) + for r in recs +) +for recs in flush_records.values(): + for r in recs: + r["t"] = r["ts"] - t0 +for recs in obs_records.values(): + for r in recs: + r["t"] = r["ts"] - t0 + +models = sorted(flush_records.keys()) +colors = ["steelblue", "tomato", "seagreen", "darkorange"] +base = LOG_FILE.rstrip("/").split("/")[-1] + +fig, axes = plt.subplots(4, 1, figsize=(14, 14), sharex=True) +fig.suptitle(f"Effective TTFT vs Real TTFT — {base}", fontsize=13) +ax_scatter, ax_median, ax_inflight, ax_ttft_stat = axes + +for i, model in enumerate(models): + color = colors[i % len(colors)] + label = model.split("/")[-1] + frecs = flush_records[model] + orecs = obs_records.get(model, []) + ft = [r["t"] for r in frecs] + fe = [r["effective"] for r in frecs] + fp = [r["p10low"] for r in frecs] + fp50 = [r["p50"] for r in frecs] + fi = [r["inflight"] for r in frecs] + + # Panel 1: scatter + ax_scatter.plot(ft, fe, color=color, linewidth=1.2, alpha=0.9, + linestyle="-", label=f"{label} effectiveTTFT") + if orecs: + ot = [r["t"] for r in orecs] + ov = [r["ttft"] for r in orecs] + ax_scatter.scatter(ot, ov, color=color, alpha=0.15, s=3, + label=f"{label} real TTFT") + + # Panel 2: binned median line + ax_median.plot(ft, fe, color=color, linewidth=1.2, alpha=0.9, + linestyle="-", label=f"{label} effectiveTTFT") + if orecs: + bins = defaultdict(list) + for r in orecs: + bins[int(r["t"] // BIN_SEC)].append(r["ttft"]) + bt = sorted(bins) + bv = [statistics.median(bins[b]) for b in bt] + ax_median.plot([b * BIN_SEC + BIN_SEC / 2 for b in bt], bv, + color=color, linewidth=1.2, alpha=0.7, + linestyle="--", label=f"{label} real TTFT (median/{BIN_SEC}s)") + + # Panel 3: inflight + ax_inflight.plot(ft, fi, color=color, alpha=0.7, linewidth=0.8, label=label) + + # Panel 4: P10Low and P50 + ax_ttft_stat.plot(ft, fp, color=color, alpha=0.9, linewidth=1.0, + linestyle="-", label=f"{label} P10Low") + ax_ttft_stat.plot(ft, fp50, color=color, alpha=0.5, linewidth=0.8, + linestyle="--", label=f"{label} P50") + +for ax, title in [ + (ax_scatter, "effectiveTTFT vs real TTFT (scatter)"), + (ax_median, f"effectiveTTFT vs real TTFT (median/{BIN_SEC}s bins)"), + (ax_inflight, "Inflight requests"), + (ax_ttft_stat, "P10Low and P50"), +]: + ax.set_ylabel("TTFT (s)" if ax != ax_inflight else "Inflight") + ax.set_ylim(bottom=0) + ax.legend(fontsize=8, ncol=2) + ax.grid(True, alpha=0.3) + ax.set_title(title, fontsize=9, loc="left", pad=3) + +ax_ttft_stat.set_xlabel("Time since first request (s)") + +plt.tight_layout() +out = base + "_ttft_cmp.png" +fig.savefig(out, dpi=150) +print("saved to", out) +plt.show() diff --git a/pkg/framework/plugins/datalayer/ttftpercentile/plugin.go b/pkg/framework/plugins/datalayer/ttftpercentile/plugin.go index 2e1bfca1..cb365c7b 100644 --- a/pkg/framework/plugins/datalayer/ttftpercentile/plugin.go +++ b/pkg/framework/plugins/datalayer/ttftpercentile/plugin.go @@ -15,7 +15,7 @@ limitations under the License. */ // Package ttftpercentile tracks per-model TTFT distributions and publishes -// P10Low, P50, and a capacity estimate for the median-ttft-scorer. +// P10Low, P50, and inflightAtP50 for the median-ttft-scorer. package ttftpercentile import ( @@ -36,39 +36,33 @@ const ( PluginType = "ttft-percentile-extractor" AttributeKey = "ttft-percentile" - balancedZoneLow = 1.5 - balancedZoneHigh = 3.0 - - defaultWindowSize = 1000 - defaultWindowAge = 5 * time.Minute - defaultMinObservations = 3 - defaultIntervalDuration = 5 * time.Second - defaultInflightEMAAlpha = 0.2 - defaultLowInflightThreshold = 2 - defaultLowLoadWindowAge = 10 * time.Minute + defaultWindowSize = 5000 + defaultWindowAge = 1 * time.Minute + defaultMinObservations = 3 + defaultIntervalDuration = 5 * time.Second + defaultInflightEMAAlpha = 0.2 + defaultLowLoadWindowAge = 1 * time.Hour ) var _ dlsrc.Extractor = &TTFTPercentileExtractor{} type TTFTPercentileExtractorConfig struct { - IntervalDuration string `json:"intervalDuration,omitempty"` - WindowSize int `json:"windowSize,omitempty"` - WindowAge string `json:"windowAge,omitempty"` - MinObservations int `json:"minObservations,omitempty"` - InflightEMAAlpha float64 `json:"inflightEmaAlpha,omitempty"` - LowInflightThreshold int `json:"lowInflightThreshold,omitempty"` - LowLoadWindowAge string `json:"lowLoadWindowAge,omitempty"` + IntervalDuration string `json:"intervalDuration,omitempty"` + WindowSize int `json:"windowSize,omitempty"` + WindowAge string `json:"windowAge,omitempty"` + MinObservations int `json:"minObservations,omitempty"` + InflightEMAAlpha float64 `json:"inflightEmaAlpha,omitempty"` + LowLoadWindowAge string `json:"lowLoadWindowAge,omitempty"` } // TTFTPercentileMetrics is written to each model's attribute store every intervalDuration. type TTFTPercentileMetrics struct { Requests int64 AvgInflight float64 - InflightAtP50 float64 // inflight_at_dispatch of the P50 observation - P10LowTTFT float64 // P10 from inflight≤lowInflightThreshold obs; queue-free floor - P10TTFT float64 // P10 from all obs - P50TTFT float64 // P50 from all obs - Capacity float64 // InflightAtP50 × P10Low / (P50 − P10Low); frozen outside balanced zone + InflightAtP50 float64 // avg inflight_at_dispatch of observations in the P40-P60 band + P10LowTTFT float64 // two-level P10: P10 of the bottom decile; hardware-floor estimate + P10TTFT float64 // P10 from all obs (short window) + P50TTFT float64 // P50 from all obs (short window) LastObservedAt int64 } @@ -83,7 +77,6 @@ type modelPercentileState struct { TTFTPercentileMetrics intervalStart time.Time tracker percentileTracker - lowLoadTracker percentileTracker pending map[string]pendingEntry avgInflightInit bool } @@ -97,41 +90,29 @@ func (s *modelPercentileState) flush(now time.Time, windowAge, lowLoadWindowAge if ok10 { s.P10TTFT = p10 } - if p10low, ok := s.lowLoadTracker.quantile(0.10, now, lowLoadWindowAge, 1); ok { + if p10low, ok := s.tracker.p10Low(now, lowLoadWindowAge, minObs); ok { s.P10LowTTFT = p10low } - p10forCap := s.P10LowTTFT - if p10forCap == 0 { - p10forCap = s.P10TTFT - } - if ok50 && p10forCap > 0 { - if ratio := p50 / p10forCap; ratio >= balancedZoneLow && ratio <= balancedZoneHigh { - if denom := p50 - p10forCap; denom > 0 { - s.Capacity = inflightAtP50 * p10forCap / denom - } - } - } s.intervalStart = now } type TTFTPercentileExtractor struct { - typedName plugin.TypedName - ds datalayer.Datastore - state map[string]*modelPercentileState - windowSize int - windowAge time.Duration - minObservations int - intervalDuration time.Duration - inflightEMAAlpha float64 - lowInflightThreshold int - lowLoadWindowAge time.Duration + typedName plugin.TypedName + ds datalayer.Datastore + state map[string]*modelPercentileState + windowSize int + windowAge time.Duration + minObservations int + intervalDuration time.Duration + inflightEMAAlpha float64 + lowLoadWindowAge time.Duration } func ExtractorFactory(name string, parameters json.RawMessage, h plugin.Handle) (plugin.Plugin, error) { cfg := TTFTPercentileExtractorConfig{ IntervalDuration: defaultIntervalDuration.String(), WindowSize: defaultWindowSize, WindowAge: defaultWindowAge.String(), MinObservations: defaultMinObservations, - InflightEMAAlpha: defaultInflightEMAAlpha, LowInflightThreshold: defaultLowInflightThreshold, + InflightEMAAlpha: defaultInflightEMAAlpha, LowLoadWindowAge: defaultLowLoadWindowAge.String(), } if len(parameters) > 0 { @@ -148,9 +129,6 @@ func ExtractorFactory(name string, parameters json.RawMessage, h plugin.Handle) if cfg.InflightEMAAlpha <= 0 || cfg.InflightEMAAlpha > 1 { return nil, fmt.Errorf("inflightEmaAlpha must be in (0,1] for plugin %q", name) } - if cfg.LowInflightThreshold < 0 { - return nil, fmt.Errorf("lowInflightThreshold must be >= 0 for plugin %q", name) - } interval, err := time.ParseDuration(cfg.IntervalDuration) if err != nil { return nil, fmt.Errorf("invalid intervalDuration %q for plugin %q: %w", cfg.IntervalDuration, name, err) @@ -167,7 +145,7 @@ func ExtractorFactory(name string, parameters json.RawMessage, h plugin.Handle) WithName(name).WithIntervalDuration(interval). WithWindow(cfg.WindowSize, windowAge, cfg.MinObservations). WithInflightEMAAlpha(cfg.InflightEMAAlpha). - WithLowLoad(cfg.LowInflightThreshold, lowLoadAge), nil + WithLowLoadWindowAge(lowLoadAge), nil } func NewTTFTPercentileExtractor(ds datalayer.Datastore) *TTFTPercentileExtractor { @@ -176,8 +154,7 @@ func NewTTFTPercentileExtractor(ds datalayer.Datastore) *TTFTPercentileExtractor ds: ds, state: make(map[string]*modelPercentileState), windowSize: defaultWindowSize, windowAge: defaultWindowAge, minObservations: defaultMinObservations, intervalDuration: defaultIntervalDuration, - inflightEMAAlpha: defaultInflightEMAAlpha, lowInflightThreshold: defaultLowInflightThreshold, - lowLoadWindowAge: defaultLowLoadWindowAge, + inflightEMAAlpha: defaultInflightEMAAlpha, lowLoadWindowAge: defaultLowLoadWindowAge, } } @@ -194,8 +171,8 @@ func (e *TTFTPercentileExtractor) WithWindow(size int, age time.Duration, minObs func (e *TTFTPercentileExtractor) WithInflightEMAAlpha(a float64) *TTFTPercentileExtractor { e.inflightEMAAlpha = a; return e } -func (e *TTFTPercentileExtractor) WithLowLoad(threshold int, age time.Duration) *TTFTPercentileExtractor { - e.lowInflightThreshold, e.lowLoadWindowAge = threshold, age; return e +func (e *TTFTPercentileExtractor) WithLowLoadWindowAge(age time.Duration) *TTFTPercentileExtractor { + e.lowLoadWindowAge = age; return e } func (e *TTFTPercentileExtractor) Extract(ctx context.Context, events []dlsrc.Event) error { @@ -218,7 +195,10 @@ func (e *TTFTPercentileExtractor) Extract(ctx context.Context, events []dlsrc.Ev inflight := s.Requests s.Requests++ if reqID := p.Request.Headers["x-request-id"]; reqID != "" { - s.pending[reqID] = pendingEntry{inflightAtDispatch: inflight, dispatchedAt: now} + s.pending[reqID] = pendingEntry{ + inflightAtDispatch: inflight, + dispatchedAt: now, + } } updated[model] = true @@ -241,11 +221,11 @@ func (e *TTFTPercentileExtractor) Extract(ctx context.Context, events []dlsrc.Ev var inflightAtDispatch int64 if entry, found := s.pending[reqID]; found && reqID != "" { inflightAtDispatch = entry.inflightAtDispatch - if inflightAtDispatch <= int64(e.lowInflightThreshold) { - s.lowLoadTracker.add(ttft, inflightAtDispatch, now) - } } s.tracker.add(ttft, inflightAtDispatch, now) + debugLogger.Info("ttft-observation", + "model", model, "ttft_s", ttft, "inflightAtDispatch", inflightAtDispatch, + ) } if reqID != "" { delete(s.pending, reqID) @@ -273,10 +253,21 @@ func (e *TTFTPercentileExtractor) Extract(ctx context.Context, events []dlsrc.Ev } m := s.TTFTPercentileMetrics e.ds.GetOrCreateModel(model).GetAttributes().Put(AttributeKey, m) + p10eff := m.P10LowTTFT + if p10eff == 0 { + p10eff = m.P10TTFT + } + eff := p10eff + if p10eff > 0 && m.InflightAtP50 > 0 && m.P50TTFT > p10eff { + eff = p10eff + float64(m.Requests)*(m.P50TTFT-p10eff)/m.InflightAtP50 + if eff < p10eff { + eff = p10eff + } + } debugLogger.Info("ttft-percentile wrote attribute", "model", model, "Requests", m.Requests, "AvgInflight", m.AvgInflight, "InflightAtP50", m.InflightAtP50, "P10Low_s", m.P10LowTTFT, - "P10_s", m.P10TTFT, "P50_s", m.P50TTFT, "Capacity", m.Capacity, + "P10_s", m.P10TTFT, "P50_s", m.P50TTFT, "EffectiveTTFT_s", eff, ) } return nil @@ -287,9 +278,8 @@ func (e *TTFTPercentileExtractor) getOrCreate(model string) *modelPercentileStat return s } s := &modelPercentileState{ - tracker: newSlidingWindowTracker(e.windowSize), - lowLoadTracker: newSlidingWindowTracker(e.windowSize), - pending: make(map[string]pendingEntry), + tracker: newSlidingWindowTracker(e.windowSize), + pending: make(map[string]pendingEntry), } e.state[model] = s return s diff --git a/pkg/framework/plugins/datalayer/ttftpercentile/tracker.go b/pkg/framework/plugins/datalayer/ttftpercentile/tracker.go index cdeb46cd..2be599f0 100644 --- a/pkg/framework/plugins/datalayer/ttftpercentile/tracker.go +++ b/pkg/framework/plugins/datalayer/ttftpercentile/tracker.go @@ -24,9 +24,14 @@ import ( type percentileTracker interface { add(value float64, inflight int64, ts time.Time) quantile(p float64, now time.Time, maxAge time.Duration, minCount int) (float64, bool) - // quantileWithInflight returns the p-th percentile TTFT and the inflight of the - // observation at that position — same request, self-consistent (TTFT, inflight) pair. + // quantileWithInflight returns the p-th percentile TTFT and the average inflight + // of observations in the [p-0.1, p+0.1] band — more stable than a single-point inflight. quantileWithInflight(p float64, now time.Time, maxAge time.Duration, minCount int) (float64, float64, bool) + // p10Low computes a two-level P10: first finds the P10 TTFT threshold from all + // observations, then returns the P10 of observations at or below that threshold. + // This isolates low-queue-wait observations without requiring a fixed inflight cut-off, + // so P10Low updates continuously regardless of sustained load level. + p10Low(now time.Time, maxAge time.Duration, minCount int) (float64, bool) } type observation struct { @@ -84,19 +89,64 @@ func (t *slidingWindowTracker) quantile(p float64, now time.Time, maxAge time.Du return vals[lo] + (idx-float64(lo))*(vals[lo+1]-vals[lo]), true } +func (t *slidingWindowTracker) p10Low(now time.Time, maxAge time.Duration, minCount int) (float64, bool) { + obs := t.recentObs(now, maxAge) + if len(obs) < minCount { + return 0, false + } + vals := make([]float64, len(obs)) + for i, o := range obs { + vals[i] = o.value + } + sort.Float64s(vals) + + // Level 1: P10 threshold — index of the 10th-percentile observation. + lo := int(0.10 * float64(len(vals)-1)) + + // Level 2: P10 of the bottom-decile slice (vals[:lo+1], already sorted). + // This is ~P1 of all observations: the fastest requests in the window + // regardless of their inflight count, approximating the hardware floor. + low := vals[:lo+1] + idx2 := 0.10 * float64(len(low)-1) + lo2 := int(idx2) + if lo2+1 >= len(low) { + return low[lo2], true + } + return low[lo2] + (idx2-float64(lo2))*(low[lo2+1]-low[lo2]), true +} + func (t *slidingWindowTracker) quantileWithInflight(p float64, now time.Time, maxAge time.Duration, minCount int) (float64, float64, bool) { obs := t.recentObs(now, maxAge) if len(obs) < minCount { return 0, 0, false } sort.Slice(obs, func(i, j int) bool { return obs[i].value < obs[j].value }) - idx := p * float64(len(obs)-1) + n := len(obs) + + idx := p * float64(n-1) lo := int(idx) var value float64 - if lo+1 >= len(obs) { + if lo+1 >= n { value = obs[lo].value } else { value = obs[lo].value + (idx-float64(lo))*(obs[lo+1].value-obs[lo].value) } - return value, float64(obs[lo].inflight), true + + // Average inflight of observations in the [p-0.1, p+0.1] band. + // Using a band rather than a single point makes inflightAtP50 more stable. + bandLo := int((p - 0.10) * float64(n-1)) + if bandLo < 0 { + bandLo = 0 + } + bandHi := int((p + 0.10) * float64(n-1)) + if bandHi >= n { + bandHi = n - 1 + } + var sum float64 + for i := bandLo; i <= bandHi; i++ { + sum += float64(obs[i].inflight) + } + avgInflight := sum / float64(bandHi-bandLo+1) + + return value, avgInflight, true } diff --git a/pkg/framework/plugins/modelselector/scorer/queuettft/README.md b/pkg/framework/plugins/modelselector/scorer/queuettft/README.md index 1ab47d0c..277b205a 100644 --- a/pkg/framework/plugins/modelselector/scorer/queuettft/README.md +++ b/pkg/framework/plugins/modelselector/scorer/queuettft/README.md @@ -6,63 +6,64 @@ Routes each request to the model with the lowest predicted TTFT under current lo Every TTFT decomposes as `TTFT = prefill_time + queue_wait`. -**P10Low** — queue-free service floor (10th percentile, inflight_at_dispatch ≤ 2): +**P10Low** — hardware-bound service floor: + +Computed from a long window (default 1h) using all observations regardless of inflight level: +1. Find the P10 TTFT threshold across all observations in the window +2. Take the P10 of only the observations at or below that threshold (~P1 of all) + +This isolates the fastest requests in the window — those with the least queue wait — without +requiring the model to have idle periods. P10Low is hardware-bound and stable: prefill time +does not change with queue depth, concurrency level, or scale events. + +**P50 and inflightAtP50** — current operating point (short window, default 1m): ``` -P10Low ≈ prefill_time +P50 = 50th percentile TTFT +inflightAtP50 = average inflight_at_dispatch of observations in the P40-P60 band ``` -High-inflight observations are excluded, so P10Low is immune to burst flooding. -A 10-min window keeps the estimate alive during sustained overload. +The short window keeps P50 responsive to current load. Averaging over a band rather than +a single observation makes inflightAtP50 more stable. -**Capacity** — updated only when `P50/P10Low ∈ [1.5, 3.0]` (balanced zone): +**effectiveTTFT** — predicted TTFT for a request arriving now: ``` -capacity = inflightAtP50 × P10Low / (P50 − P10Low) +effectiveTTFT = P10Low + inflight x (P50 - P10Low) / inflightAtP50 ``` -Derived from `P50 - P10Low = P10Low * inflight/capacity`. where `P50 - P10Low` is estimating the waiting time. -`inflightAtP50` is the `inflight_at_dispatch` of the observation whose TTFT landed at -P50. Below 1.5 the denominator is too noisy; above 3.0, P50 is contaminated by flooding; capacity is frozen in both cases. +Falls back to P10Low when P50 is not yet available or equals the floor. -**Scorer:** +**Score:** ``` -loadRatio = inflight / capacity -effectiveTTFT = P10Low × (1 + (loadRatio − 1)) when loadRatio > 1 - = P10Low otherwise -score = (maxTTFT − effectiveTTFT) / (maxTTFT − minTTFT) +score = (maxTTFT - effectiveTTFT) / (maxTTFT - minTTFT) ``` -Within capacity: `effectiveTTFT = P10Low`, the queue-free baseline. -At 2× capacity: `effectiveTTFT = 2 × P10Low` — exactly what the queue model predicts. Unobserved models score 1.0 (cold start) or 0.5 (idle alongside observed peers). -## Why it should work physically +## Why it works physically -Think of the model as C parallel slots, each finishing a request in P10Low seconds. -With N requests in the system, the server clears them at a rate of C every P10Low, so -draining the current backlog takes `(N / C) × P10Low`. That drain time is the queue -wait a new request sees: +Think of the model as having C parallel slots, each taking P10Low seconds per request. +With N requests in flight, a new arrival waits for the backlog to drain: ``` -TTFT = P10Low + (N / C) × P10Low = P10Low × (1 + N / C) +TTFT = P10Low + (N / C) x P10Low = P10Low x (1 + N / C) ``` -Rearranging for C at the observed operating point `(inflightAtP50, P50)`: +This is a straight line through `(0, P10Low)` with slope `P10Low / C`. The scorer anchors +this line at two observed points instead of estimating C explicitly: -``` -capacity C = inflightAtP50 × P10Low / (P50 − P10Low) -``` +- at `inflight = 0`: TTFT = P10Low (hardware floor, no queue) +- at `inflight = inflightAtP50`: TTFT = P50 (observed median operating point) -The same model gives the scorer's penalty directly: when `inflight > C`, the predicted -drain time exceeds P10Low, so `effectiveTTFT = P10Low × (inflight / C)`. +The formula extrapolates this line to the current inflight, giving the predicted TTFT for +a new request. No capacity variable, no regression, no tunable parameters. -## Possible Enhancements -### prompt-length awareness +## Parameters -Prefill time scales roughly linearly with token count, but the scorer currently treats -all requests identically regardless of prompt size. The fix is to normalise TTFT by -character count in the tracker (`rate = TTFT / chars`) and scale at score time: -``` -effectiveTTFT = P10Low_rate × request_chars × max(1, inflight/capacity) -``` -This makes the scorer aware of the current request's weight without requiring a -model-specific tokeniser; character count (≈ tokens / 4) is a sufficient proxy. +| Parameter | Default | Description | +|---|---|---| +| `windowAge` | 1m | Window for P50 (short -- keeps P50 fresh and responsive) | +| `lowLoadWindowAge` | 1h | Window for two-level P10Low (long -- stable hardware floor) | +| `windowSize` | 5000 | Ring buffer capacity (~200 KB per model) | +| `minObservations` | 3 | Minimum observations required to compute any percentile | + +## Possible Enhancements ### Score-proportional picker @@ -71,6 +72,6 @@ score difference into a full traffic flip. This causes oscillation: the best mod overloads, all traffic switches to the other, the first model drains and wins again. `score-proportional-picker` eliminates this by routing probabilistically: ``` -P(model i) ∝ score_i^(1/T) # T = temperature, default 1.0 +P(model i) proportional to score_i^(1/T) # T = temperature, default 1.0 ``` -At T = 1.0, a model scoring 0.8 vs 0.2 receives ≈ 80% vs 20% of requests. +At T = 1.0, a model scoring 0.8 vs 0.2 receives ~80% vs 20% of requests. diff --git a/pkg/framework/plugins/modelselector/scorer/queuettft/plugin.go b/pkg/framework/plugins/modelselector/scorer/queuettft/plugin.go index bee478ec..767bca82 100644 --- a/pkg/framework/plugins/modelselector/scorer/queuettft/plugin.go +++ b/pkg/framework/plugins/modelselector/scorer/queuettft/plugin.go @@ -15,14 +15,13 @@ limitations under the License. */ // Package medianttft scores models by predicted TTFT under current load. -// Model P10Low × (1 + inflight/capacity) is used to predict queue wait, -// where capacity is estimated from paired (P50, inflightAtP50) observations. +// effectiveTTFT = P10Low + inflight × (P50 − P10Low) / inflightAtP50: +// a line through (0, P10Low) and (inflightAtP50, P50), extrapolated linearly. package medianttft import ( "context" "encoding/json" - "fmt" "math" "sigs.k8s.io/controller-runtime/pkg/log" @@ -35,46 +34,21 @@ import ( "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/plugins/datalayer/ttftpercentile" ) -const ( - PluginType = "median-ttft-scorer" - defaultInflightPenaltyWeight = 1.0 -) +const PluginType = "median-ttft-scorer" var _ modelselector.Scorer = &MedianTTFTScorer{} -type MedianTTFTScorerConfig struct { - // InflightPenaltyWeight scales the overload penalty: - // effectiveTTFT = P10Low × (1 + weight × (inflight/capacity − 1)) - // 0 disables the penalty; 1.0 (default) equals the queue model prediction. - InflightPenaltyWeight *float64 `json:"inflightPenaltyWeight,omitempty"` -} - type MedianTTFTScorer struct { - typedName plugin.TypedName - inflightPenaltyWeight float64 + typedName plugin.TypedName } -func ScorerFactory(name string, parameters json.RawMessage, _ plugin.Handle) (plugin.Plugin, error) { - cfg := MedianTTFTScorerConfig{} - if len(parameters) > 0 { - if err := json.Unmarshal(parameters, &cfg); err != nil { - return nil, fmt.Errorf("failed to parse parameters for plugin %q: %w", name, err) - } - } - w := defaultInflightPenaltyWeight - if cfg.InflightPenaltyWeight != nil { - if *cfg.InflightPenaltyWeight < 0 { - return nil, fmt.Errorf("inflightPenaltyWeight must be >= 0 for plugin %q", name) - } - w = *cfg.InflightPenaltyWeight - } - return NewMedianTTFTScorer().WithName(name).WithInflightPenaltyWeight(w), nil +func ScorerFactory(name string, _ json.RawMessage, _ plugin.Handle) (plugin.Plugin, error) { + return NewMedianTTFTScorer().WithName(name), nil } func NewMedianTTFTScorer() *MedianTTFTScorer { return &MedianTTFTScorer{ - typedName: plugin.TypedName{Type: PluginType, Name: PluginType}, - inflightPenaltyWeight: defaultInflightPenaltyWeight, + typedName: plugin.TypedName{Type: PluginType, Name: PluginType}, } } @@ -82,9 +56,6 @@ func (s *MedianTTFTScorer) TypedName() plugin.TypedName { return s.typedName } func (s *MedianTTFTScorer) WithName(name string) *MedianTTFTScorer { s.typedName.Name = name; return s } -func (s *MedianTTFTScorer) WithInflightPenaltyWeight(w float64) *MedianTTFTScorer { - s.inflightPenaltyWeight = w; return s -} // Score returns (maxTTFT − effectiveTTFT) / (maxTTFT − minTTFT) per model. // Unobserved models score 1.0 (all unobserved) or 0.5 (some peers observed). @@ -147,20 +118,18 @@ func (s *MedianTTFTScorer) effectiveTTFT(ctx context.Context, model datalayer.Mo return 0 } + // effectiveTTFT = P10Low + inflight × (P50 − P10Low) / inflightAtP50 + // Falls back to P10Low when P50 is not yet available or equals the floor. eff := p10 - var loadRatio float64 - if m.Capacity >= 1 { - loadRatio = float64(m.Requests) / m.Capacity - if loadRatio > 1 { - eff = p10 * (1 + s.inflightPenaltyWeight*(loadRatio-1)) - } + if m.InflightAtP50 > 0 && m.P50TTFT > p10 { + eff = p10 + float64(m.Requests)*(m.P50TTFT-p10)/m.InflightAtP50 } if dl := log.FromContext(ctx).V(logutil.DEBUG); dl.Enabled() { dl.Info("median-ttft effective", - "model", model.GetName(), "inflight", m.Requests, "capacity", m.Capacity, - "loadRatio", loadRatio, "P10Low_s", m.P10LowTTFT, "P10_s", m.P10TTFT, - "P50_s", m.P50TTFT, "effectiveTTFT", eff, + "model", model.GetName(), "inflight", m.Requests, + "inflightAtP50", m.InflightAtP50, "P10Low_s", m.P10LowTTFT, + "P10_s", m.P10TTFT, "P50_s", m.P50TTFT, "effectiveTTFT", eff, ) } return eff diff --git a/pkg/handlers/server.go b/pkg/handlers/server.go index 3cab9c26..960c15e9 100644 --- a/pkg/handlers/server.go +++ b/pkg/handlers/server.go @@ -172,7 +172,9 @@ func (s *Server) Process(srv extProcPb.ExternalProcessor_ProcessServer) error { responses = s.HandleResponseHeaders(ctx, reqCtx, v.ResponseHeaders) loggerVerbose.Info("processing response headers complete") case *extProcPb.ProcessingRequest_ResponseBody: - loggerVerbose.Info("Incoming response body chunk", "EoS", v.ResponseBody.EndOfStream) + // Logged at TRACE: one line per stream chunk is a firehose under load (≈36 lines/request) + // that drowns scorer/extractor DEBUG logs and triggers kubelet log rotation. Use --v=5 to see it. + logger.V(logutil.TRACE).Info("Incoming response body chunk", "EoS", v.ResponseBody.EndOfStream) if reqCtx.ResponseFirstChunkTimestamp.IsZero() { reqCtx.ResponseFirstChunkTimestamp = time.Now() } From 0842866f4abbc116124f375529cdda9abd262e09 Mon Sep 17 00:00:00 2001 From: Mohammad Date: Tue, 23 Jun 2026 01:19:17 +0300 Subject: [PATCH 04/27] Rename the scorer. Signed-off-by: Mohammad --- cmd/runner/runner.go | 4 +-- examples/medianttft-values.yaml | 2 +- .../datalayer/ttftpercentile/plugin.go | 28 ++++++------------- .../modelselector/scorer/queuettft/README.md | 4 +-- .../modelselector/scorer/queuettft/plugin.go | 28 +++++++++---------- 5 files changed, 27 insertions(+), 39 deletions(-) diff --git a/cmd/runner/runner.go b/cmd/runner/runner.go index 3b647a61..1563a7af 100644 --- a/cmd/runner/runner.go +++ b/cmd/runner/runner.go @@ -54,7 +54,7 @@ import ( "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/plugins/modelselector/picker/random" "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/plugins/modelselector/picker/weightedrandom" inflightrequestsscorer "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/plugins/modelselector/scorer/inflightrequests" - medianttftscorer "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/plugins/modelselector/scorer/queuettft" + queuettftscorer "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/plugins/modelselector/scorer/queuettft" "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/plugins/requesthandling/basemodelextractor" "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/plugins/requesthandling/bodyfieldtoheader" modelselectorplugin "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/plugins/requesthandling/modelselector" @@ -288,7 +288,7 @@ func (r *Runner) registerInTreePlugins() { plugin.Register(weightedrandom.WeightedRandomPickerType, weightedrandom.WeightedRandomPickerFactory) plugin.Register(modelselectorplugin.ModelSelectorPluginType, modelselectorplugin.ModelSelectorPluginFactory) plugin.Register(inflightrequestsscorer.PluginType, inflightrequestsscorer.ScorerFactory) - plugin.Register(medianttftscorer.PluginType, medianttftscorer.ScorerFactory) + plugin.Register(queuettftscorer.PluginType, queuettftscorer.ScorerFactory) } // registerHealthServer adds the Health gRPC server as a Runnable to the given manager. diff --git a/examples/medianttft-values.yaml b/examples/medianttft-values.yaml index 5ee2daa5..83e8eb31 100644 --- a/examples/medianttft-values.yaml +++ b/examples/medianttft-values.yaml @@ -10,7 +10,7 @@ payloadProcessor: headerName: X-Gateway-Model-Name - type: base-model-to-header - type: model-selector - - type: median-ttft-scorer + - type: queue-ttft-scorer # effectiveTTFT = P10Low + inflight x (P50 - P10Low) / inflightAtP50 # line through (0, P10Low) and (inflightAtP50, P50), extrapolated linearly - type: max-score-picker diff --git a/pkg/framework/plugins/datalayer/ttftpercentile/plugin.go b/pkg/framework/plugins/datalayer/ttftpercentile/plugin.go index cb365c7b..776d2d42 100644 --- a/pkg/framework/plugins/datalayer/ttftpercentile/plugin.go +++ b/pkg/framework/plugins/datalayer/ttftpercentile/plugin.go @@ -57,13 +57,12 @@ type TTFTPercentileExtractorConfig struct { // TTFTPercentileMetrics is written to each model's attribute store every intervalDuration. type TTFTPercentileMetrics struct { - Requests int64 - AvgInflight float64 - InflightAtP50 float64 // avg inflight_at_dispatch of observations in the P40-P60 band - P10LowTTFT float64 // two-level P10: P10 of the bottom decile; hardware-floor estimate - P10TTFT float64 // P10 from all obs (short window) - P50TTFT float64 // P50 from all obs (short window) - LastObservedAt int64 + Requests int64 + AvgInflight float64 + InflightAtP50 float64 // avg inflight_at_dispatch of observations in the P40-P60 band + P10LowTTFT float64 // two-level P10: P10 of the bottom decile; hardware-floor estimate + P10TTFT float64 // P10 from all obs (short window) + P50TTFT float64 // P50 from all obs (short window) } func (m TTFTPercentileMetrics) Clone() datalayer.Cloneable { return m } @@ -85,7 +84,7 @@ func (s *modelPercentileState) flush(now time.Time, windowAge, lowLoadWindowAge p50, inflightAtP50, ok50 := s.tracker.quantileWithInflight(0.50, now, windowAge, minObs) p10, ok10 := s.tracker.quantile(0.10, now, windowAge, minObs) if ok50 { - s.P50TTFT, s.InflightAtP50, s.LastObservedAt = p50, inflightAtP50, now.UnixNano() + s.P50TTFT, s.InflightAtP50 = p50, inflightAtP50 } if ok10 { s.P10TTFT = p10 @@ -253,21 +252,10 @@ func (e *TTFTPercentileExtractor) Extract(ctx context.Context, events []dlsrc.Ev } m := s.TTFTPercentileMetrics e.ds.GetOrCreateModel(model).GetAttributes().Put(AttributeKey, m) - p10eff := m.P10LowTTFT - if p10eff == 0 { - p10eff = m.P10TTFT - } - eff := p10eff - if p10eff > 0 && m.InflightAtP50 > 0 && m.P50TTFT > p10eff { - eff = p10eff + float64(m.Requests)*(m.P50TTFT-p10eff)/m.InflightAtP50 - if eff < p10eff { - eff = p10eff - } - } debugLogger.Info("ttft-percentile wrote attribute", "model", model, "Requests", m.Requests, "AvgInflight", m.AvgInflight, "InflightAtP50", m.InflightAtP50, "P10Low_s", m.P10LowTTFT, - "P10_s", m.P10TTFT, "P50_s", m.P50TTFT, "EffectiveTTFT_s", eff, + "P10_s", m.P10TTFT, "P50_s", m.P50TTFT, ) } return nil diff --git a/pkg/framework/plugins/modelselector/scorer/queuettft/README.md b/pkg/framework/plugins/modelselector/scorer/queuettft/README.md index 277b205a..fb3e178e 100644 --- a/pkg/framework/plugins/modelselector/scorer/queuettft/README.md +++ b/pkg/framework/plugins/modelselector/scorer/queuettft/README.md @@ -1,4 +1,4 @@ -# Median-TTFT Scorer +# Queue-TTFT Scorer Routes each request to the model with the lowest predicted TTFT under current load. @@ -10,7 +10,7 @@ Every TTFT decomposes as `TTFT = prefill_time + queue_wait`. Computed from a long window (default 1h) using all observations regardless of inflight level: 1. Find the P10 TTFT threshold across all observations in the window -2. Take the P10 of only the observations at or below that threshold (~P1 of all) +2. Take the P10 of only the observations at or below that threshold This isolates the fastest requests in the window — those with the least queue wait — without requiring the model to have idle periods. P10Low is hardware-bound and stable: prefill time diff --git a/pkg/framework/plugins/modelselector/scorer/queuettft/plugin.go b/pkg/framework/plugins/modelselector/scorer/queuettft/plugin.go index 767bca82..b579b804 100644 --- a/pkg/framework/plugins/modelselector/scorer/queuettft/plugin.go +++ b/pkg/framework/plugins/modelselector/scorer/queuettft/plugin.go @@ -14,10 +14,10 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Package medianttft scores models by predicted TTFT under current load. +// Package queuettft scores models by predicted TTFT under current load. // effectiveTTFT = P10Low + inflight × (P50 − P10Low) / inflightAtP50: // a line through (0, P10Low) and (inflightAtP50, P50), extrapolated linearly. -package medianttft +package queuettft import ( "context" @@ -34,32 +34,32 @@ import ( "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/plugins/datalayer/ttftpercentile" ) -const PluginType = "median-ttft-scorer" +const PluginType = "queue-ttft-scorer" -var _ modelselector.Scorer = &MedianTTFTScorer{} +var _ modelselector.Scorer = &QueueTTFTScorer{} -type MedianTTFTScorer struct { +type QueueTTFTScorer struct { typedName plugin.TypedName } func ScorerFactory(name string, _ json.RawMessage, _ plugin.Handle) (plugin.Plugin, error) { - return NewMedianTTFTScorer().WithName(name), nil + return NewQueueTTFTScorer().WithName(name), nil } -func NewMedianTTFTScorer() *MedianTTFTScorer { - return &MedianTTFTScorer{ +func NewQueueTTFTScorer() *QueueTTFTScorer { + return &QueueTTFTScorer{ typedName: plugin.TypedName{Type: PluginType, Name: PluginType}, } } -func (s *MedianTTFTScorer) TypedName() plugin.TypedName { return s.typedName } -func (s *MedianTTFTScorer) WithName(name string) *MedianTTFTScorer { +func (s *QueueTTFTScorer) TypedName() plugin.TypedName { return s.typedName } +func (s *QueueTTFTScorer) WithName(name string) *QueueTTFTScorer { s.typedName.Name = name; return s } // Score returns (maxTTFT − effectiveTTFT) / (maxTTFT − minTTFT) per model. // Unobserved models score 1.0 (all unobserved) or 0.5 (some peers observed). -func (s *MedianTTFTScorer) Score(ctx context.Context, _ *plugin.CycleState, _ *requesthandling.InferenceRequest, models []datalayer.Model) map[datalayer.Model]float64 { +func (s *QueueTTFTScorer) Score(ctx context.Context, _ *plugin.CycleState, _ *requesthandling.InferenceRequest, models []datalayer.Model) map[datalayer.Model]float64 { ttfts := make(map[datalayer.Model]float64, len(models)) minTTFT, maxTTFT := math.MaxFloat64, 0.0 allUnobserved := true @@ -95,13 +95,13 @@ func (s *MedianTTFTScorer) Score(ctx context.Context, _ *plugin.CycleState, _ *r if dl := log.FromContext(ctx).V(logutil.DEBUG); dl.Enabled() { for _, model := range models { - dl.Info("median-ttft score", "model", model.GetName(), "effectiveTTFT", ttfts[model], "score", scores[model]) + dl.Info("queue-ttft score", "model", model.GetName(), "effectiveTTFT", ttfts[model], "score", scores[model]) } } return scores } -func (s *MedianTTFTScorer) effectiveTTFT(ctx context.Context, model datalayer.Model) float64 { +func (s *QueueTTFTScorer) effectiveTTFT(ctx context.Context, model datalayer.Model) float64 { val, ok := model.GetAttributes().Get(ttftpercentile.AttributeKey) if !ok { return 0 @@ -126,7 +126,7 @@ func (s *MedianTTFTScorer) effectiveTTFT(ctx context.Context, model datalayer.Mo } if dl := log.FromContext(ctx).V(logutil.DEBUG); dl.Enabled() { - dl.Info("median-ttft effective", + dl.Info("queue-ttft effective", "model", model.GetName(), "inflight", m.Requests, "inflightAtP50", m.InflightAtP50, "P10Low_s", m.P10LowTTFT, "P10_s", m.P10TTFT, "P50_s", m.P50TTFT, "effectiveTTFT", eff, From 64936e10f69c9fa6e7302f752d5de83121ce1802 Mon Sep 17 00:00:00 2001 From: Mohammad Date: Tue, 23 Jun 2026 12:16:01 +0300 Subject: [PATCH 05/27] Update readme. Signed-off-by: Mohammad --- examples/medianttft-values.yaml | 8 +++----- .../modelselector/scorer/queuettft/README.md | 20 ++++++++----------- 2 files changed, 11 insertions(+), 17 deletions(-) diff --git a/examples/medianttft-values.yaml b/examples/medianttft-values.yaml index 83e8eb31..53c9aa46 100644 --- a/examples/medianttft-values.yaml +++ b/examples/medianttft-values.yaml @@ -11,16 +11,14 @@ payloadProcessor: - type: base-model-to-header - type: model-selector - type: queue-ttft-scorer - # effectiveTTFT = P10Low + inflight x (P50 - P10Low) / inflightAtP50 - # line through (0, P10Low) and (inflightAtP50, P50), extrapolated linearly - type: max-score-picker - type: ttft-percentile-extractor parameters: intervalDuration: 1s - windowSize: 5000 # ~200 KB per model; holds ~1h at 1 req/s or ~8 min at 10 req/s - windowAge: 1m # window for P50 and P10 (short = fresh, responsive) + windowSize: 5000 + windowAge: 1m minObservations: 3 - inflightEmaAlpha: 0.2 # EMA of inflight count (observability only) + inflightEmaAlpha: 0.2 lowLoadWindowAge: 1h # window for two-level P10Low (long = stable hardware floor) - type: model-config-datasource parameters: diff --git a/pkg/framework/plugins/modelselector/scorer/queuettft/README.md b/pkg/framework/plugins/modelselector/scorer/queuettft/README.md index fb3e178e..e7c27f41 100644 --- a/pkg/framework/plugins/modelselector/scorer/queuettft/README.md +++ b/pkg/framework/plugins/modelselector/scorer/queuettft/README.md @@ -38,21 +38,17 @@ Unobserved models score 1.0 (cold start) or 0.5 (idle alongside observed peers). ## Why it works physically -Think of the model as having C parallel slots, each taking P10Low seconds per request. -With N requests in flight, a new arrival waits for the backlog to drain: +When more requests are in flight, a new request has to wait longer in the queue before +the model processes it. The longer the queue, the higher the TTFT. This wait grows +roughly in proportion to the number of in-flight requests. -``` -TTFT = P10Low + (N / C) x P10Low = P10Low x (1 + N / C) -``` - -This is a straight line through `(0, P10Low)` with slope `P10Low / C`. The scorer anchors -this line at two observed points instead of estimating C explicitly: +The scorer draws a straight line through two points it has actually observed: -- at `inflight = 0`: TTFT = P10Low (hardware floor, no queue) -- at `inflight = inflightAtP50`: TTFT = P50 (observed median operating point) +- when there is no queue (`inflight = 0`): TTFT = P10Low (just the raw prefill time) +- at the recent median load (`inflight = inflightAtP50`): TTFT = P50 -The formula extrapolates this line to the current inflight, giving the predicted TTFT for -a new request. No capacity variable, no regression, no tunable parameters. +It then reads off that line at the current inflight to predict what the next request +will wait. No fitting, no tunable parameters — just two observed points. ## Parameters From 2e674bd04de6f5ffae9afe34f993f34bc6946e93 Mon Sep 17 00:00:00 2001 From: Mohammad Date: Tue, 23 Jun 2026 18:39:29 +0300 Subject: [PATCH 06/27] Add model config changes. Signed-off-by: Mohammad --- .../payload-processor/templates/config.yaml | 7 ++ examples/plot_capacity.py | 65 ++++++++++++------- .../datalayer/ttftpercentile/plugin.go | 28 +++++--- .../modelselector/scorer/queuettft/README.md | 10 ++- 4 files changed, 74 insertions(+), 36 deletions(-) diff --git a/config/charts/payload-processor/templates/config.yaml b/config/charts/payload-processor/templates/config.yaml index 3ce225af..4785d58d 100644 --- a/config/charts/payload-processor/templates/config.yaml +++ b/config/charts/payload-processor/templates/config.yaml @@ -26,4 +26,11 @@ data: kind: PayloadProcessorConfig {{- .Values.payloadProcessor.customConfig | toYaml | nindent 4 }} {{- end }} + {{- if .Values.payloadProcessor.listModels }} + {{- $models := list }} + {{- range .Values.payloadProcessor.listModels }} + {{- $models = append $models (dict "name" .) }} + {{- end }} + models.json: {{ dict "models" $models | toJson | quote }} + {{- end }} --- diff --git a/examples/plot_capacity.py b/examples/plot_capacity.py index 90b8e5e1..7f416aec 100644 --- a/examples/plot_capacity.py +++ b/examples/plot_capacity.py @@ -17,8 +17,9 @@ import matplotlib.pyplot as plt -LOG_FILE = sys.argv[1] if len(sys.argv) > 1 else "logs/bench_median33" -BIN_SEC = 10 # bucket width for the median graph +LOG_FILE = sys.argv[1] if len(sys.argv) > 1 else "logs/bench_median33" +MAX_T_SEC = float(sys.argv[2]) if len(sys.argv) > 2 else None # optional time cutoff +BIN_SEC = 10 # bucket width for the median graph # --------------------------------------------------------------------------- # Parse @@ -41,16 +42,18 @@ continue if msg == "ttft-percentile wrote attribute": flush_records[model].append({ - "ts": obj["ts"], - "inflight": obj.get("Requests", 0), - "p10low": obj.get("P10Low_s", 0), - "p50": obj.get("P50_s", 0), - "effective":obj.get("EffectiveTTFT_s", 0), + "ts": obj["ts"], + "inflight": obj.get("Requests", 0), + "inflight_at_p50": obj.get("InflightAtP50", 0), + "p10low": obj.get("P10Low_s", 0), + "p50": obj.get("P50_s", 0), + "effective": obj.get("EffectiveTTFT_s", 0), }) elif msg == "ttft-observation": + ttft = obj.get("ttft_s", 0) obs_records[model].append({ - "ts": obj["ts"], - "ttft": obj.get("ttft_s", 0), + "ts": obj["ts"] - ttft, # align to dispatch time (response_ts - ttft) + "ttft": ttft, }) if not flush_records: @@ -70,24 +73,33 @@ for r in recs: r["t"] = r["ts"] - t0 +if MAX_T_SEC is not None: + flush_records = {m: [r for r in recs if r["t"] <= MAX_T_SEC] for m, recs in flush_records.items()} + obs_records = {m: [r for r in recs if r["t"] <= MAX_T_SEC] for m, recs in obs_records.items()} + models = sorted(flush_records.keys()) colors = ["steelblue", "tomato", "seagreen", "darkorange"] base = LOG_FILE.rstrip("/").split("/")[-1] -fig, axes = plt.subplots(4, 1, figsize=(14, 14), sharex=True) +fig, axes = plt.subplots(6, 1, figsize=(14, 20), sharex=True) fig.suptitle(f"Effective TTFT vs Real TTFT — {base}", fontsize=13) -ax_scatter, ax_median, ax_inflight, ax_ttft_stat = axes +ax_scatter, ax_median, ax_inflight, ax_ttft_stat, ax_load_ratio, ax_spread = axes for i, model in enumerate(models): color = colors[i % len(colors)] label = model.split("/")[-1] frecs = flush_records[model] orecs = obs_records.get(model, []) - ft = [r["t"] for r in frecs] - fe = [r["effective"] for r in frecs] - fp = [r["p10low"] for r in frecs] - fp50 = [r["p50"] for r in frecs] - fi = [r["inflight"] for r in frecs] + ft = [r["t"] for r in frecs] + fe = [r["effective"] for r in frecs] + fp = [r["p10low"] for r in frecs] + fp50 = [r["p50"] for r in frecs] + fi = [r["inflight"] for r in frecs] + fip50 = [r["inflight_at_p50"] for r in frecs] + # load ratio: inflight / inflightAtP50 (capped to avoid div-by-zero noise) + fratio = [inf / ip50 if ip50 > 0 else 0 for inf, ip50 in zip(fi, fip50)] + # spread: P50 - P10Low (the TTFT range the formula scales over) + fspread = [p50 - p10 for p50, p10 in zip(fp50, fp)] # Panel 1: scatter ax_scatter.plot(ft, fe, color=color, linewidth=1.2, alpha=0.9, @@ -120,13 +132,22 @@ ax_ttft_stat.plot(ft, fp50, color=color, alpha=0.5, linewidth=0.8, linestyle="--", label=f"{label} P50") -for ax, title in [ - (ax_scatter, "effectiveTTFT vs real TTFT (scatter)"), - (ax_median, f"effectiveTTFT vs real TTFT (median/{BIN_SEC}s bins)"), - (ax_inflight, "Inflight requests"), - (ax_ttft_stat, "P10Low and P50"), + # Panel 5: load ratio = inflight / inflightAtP50 + ax_load_ratio.plot(ft, fratio, color=color, linewidth=1.0, alpha=0.85, label=label) + ax_load_ratio.axhline(1.0, color="gray", linewidth=0.7, linestyle=":") + + # Panel 6: spread = P50 - P10Low + ax_spread.plot(ft, fspread, color=color, linewidth=1.0, alpha=0.85, label=label) + +for ax, title, ylabel in [ + (ax_scatter, "effectiveTTFT vs real TTFT (scatter)", "TTFT (s)"), + (ax_median, f"effectiveTTFT vs real TTFT (median/{BIN_SEC}s bins)", "TTFT (s)"), + (ax_inflight, "Inflight requests", "Inflight"), + (ax_ttft_stat, "P10Low and P50", "TTFT (s)"), + (ax_load_ratio, "Load ratio: inflight / inflightAtP50 (1.0 = median load)", "ratio"), + (ax_spread, "Spread: P50 − P10Low (TTFT range the slope scales over)", "TTFT (s)"), ]: - ax.set_ylabel("TTFT (s)" if ax != ax_inflight else "Inflight") + ax.set_ylabel(ylabel) ax.set_ylim(bottom=0) ax.legend(fontsize=8, ncol=2) ax.grid(True, alpha=0.3) diff --git a/pkg/framework/plugins/datalayer/ttftpercentile/plugin.go b/pkg/framework/plugins/datalayer/ttftpercentile/plugin.go index 776d2d42..cb365c7b 100644 --- a/pkg/framework/plugins/datalayer/ttftpercentile/plugin.go +++ b/pkg/framework/plugins/datalayer/ttftpercentile/plugin.go @@ -57,12 +57,13 @@ type TTFTPercentileExtractorConfig struct { // TTFTPercentileMetrics is written to each model's attribute store every intervalDuration. type TTFTPercentileMetrics struct { - Requests int64 - AvgInflight float64 - InflightAtP50 float64 // avg inflight_at_dispatch of observations in the P40-P60 band - P10LowTTFT float64 // two-level P10: P10 of the bottom decile; hardware-floor estimate - P10TTFT float64 // P10 from all obs (short window) - P50TTFT float64 // P50 from all obs (short window) + Requests int64 + AvgInflight float64 + InflightAtP50 float64 // avg inflight_at_dispatch of observations in the P40-P60 band + P10LowTTFT float64 // two-level P10: P10 of the bottom decile; hardware-floor estimate + P10TTFT float64 // P10 from all obs (short window) + P50TTFT float64 // P50 from all obs (short window) + LastObservedAt int64 } func (m TTFTPercentileMetrics) Clone() datalayer.Cloneable { return m } @@ -84,7 +85,7 @@ func (s *modelPercentileState) flush(now time.Time, windowAge, lowLoadWindowAge p50, inflightAtP50, ok50 := s.tracker.quantileWithInflight(0.50, now, windowAge, minObs) p10, ok10 := s.tracker.quantile(0.10, now, windowAge, minObs) if ok50 { - s.P50TTFT, s.InflightAtP50 = p50, inflightAtP50 + s.P50TTFT, s.InflightAtP50, s.LastObservedAt = p50, inflightAtP50, now.UnixNano() } if ok10 { s.P10TTFT = p10 @@ -252,10 +253,21 @@ func (e *TTFTPercentileExtractor) Extract(ctx context.Context, events []dlsrc.Ev } m := s.TTFTPercentileMetrics e.ds.GetOrCreateModel(model).GetAttributes().Put(AttributeKey, m) + p10eff := m.P10LowTTFT + if p10eff == 0 { + p10eff = m.P10TTFT + } + eff := p10eff + if p10eff > 0 && m.InflightAtP50 > 0 && m.P50TTFT > p10eff { + eff = p10eff + float64(m.Requests)*(m.P50TTFT-p10eff)/m.InflightAtP50 + if eff < p10eff { + eff = p10eff + } + } debugLogger.Info("ttft-percentile wrote attribute", "model", model, "Requests", m.Requests, "AvgInflight", m.AvgInflight, "InflightAtP50", m.InflightAtP50, "P10Low_s", m.P10LowTTFT, - "P10_s", m.P10TTFT, "P50_s", m.P50TTFT, + "P10_s", m.P10TTFT, "P50_s", m.P50TTFT, "EffectiveTTFT_s", eff, ) } return nil diff --git a/pkg/framework/plugins/modelselector/scorer/queuettft/README.md b/pkg/framework/plugins/modelselector/scorer/queuettft/README.md index e7c27f41..176988e3 100644 --- a/pkg/framework/plugins/modelselector/scorer/queuettft/README.md +++ b/pkg/framework/plugins/modelselector/scorer/queuettft/README.md @@ -12,11 +12,9 @@ Computed from a long window (default 1h) using all observations regardless of in 1. Find the P10 TTFT threshold across all observations in the window 2. Take the P10 of only the observations at or below that threshold -This isolates the fastest requests in the window — those with the least queue wait — without -requiring the model to have idle periods. P10Low is hardware-bound and stable: prefill time -does not change with queue depth, concurrency level, or scale events. +This isolates the fastest requests in the window — those with the least queue wait — without requiring the model to have idle periods. P10Low is hardware-bound and stable: prefill time does not change with queue depth, concurrency level, or scale events. -**P50 and inflightAtP50** — current operating point (short window, default 1m): +**P50 and inflightAtP50** — current operating point (short window, default 3m): ``` P50 = 50th percentile TTFT inflightAtP50 = average inflight_at_dispatch of observations in the P40-P60 band @@ -48,13 +46,13 @@ The scorer draws a straight line through two points it has actually observed: - at the recent median load (`inflight = inflightAtP50`): TTFT = P50 It then reads off that line at the current inflight to predict what the next request -will wait. No fitting, no tunable parameters — just two observed points. +will wait. ## Parameters | Parameter | Default | Description | |---|---|---| -| `windowAge` | 1m | Window for P50 (short -- keeps P50 fresh and responsive) | +| `windowAge` | 3m | Window for P50 (short -- keeps P50 fresh and responsive) | | `lowLoadWindowAge` | 1h | Window for two-level P10Low (long -- stable hardware floor) | | `windowSize` | 5000 | Ring buffer capacity (~200 KB per model) | | `minObservations` | 3 | Minimum observations required to compute any percentile | From 084ad449f5d5c27482c8bf10e60f15a4e97b17d5 Mon Sep 17 00:00:00 2001 From: Mohammad Date: Wed, 24 Jun 2026 02:18:16 +0300 Subject: [PATCH 07/27] Update Readme.md. Signed-off-by: Mohammad --- .../modelselector/scorer/queuettft/README.md | 25 +++++++++++++++---- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/pkg/framework/plugins/modelselector/scorer/queuettft/README.md b/pkg/framework/plugins/modelselector/scorer/queuettft/README.md index 176988e3..6d94345c 100644 --- a/pkg/framework/plugins/modelselector/scorer/queuettft/README.md +++ b/pkg/framework/plugins/modelselector/scorer/queuettft/README.md @@ -6,13 +6,16 @@ Routes each request to the model with the lowest predicted TTFT under current lo Every TTFT decomposes as `TTFT = prefill_time + queue_wait`. -**P10Low** — hardware-bound service floor: +**P10Low** — load-invariant service floor: Computed from a long window (default 1h) using all observations regardless of inflight level: 1. Find the P10 TTFT threshold across all observations in the window 2. Take the P10 of only the observations at or below that threshold -This isolates the fastest requests in the window — those with the least queue wait — without requiring the model to have idle periods. P10Low is hardware-bound and stable: prefill time does not change with queue depth, concurrency level, or scale events. +This isolates the fastest requests in the window — those with the least queue wait — without requiring the model to have idle periods. P10Low is invariant to load: it does not change with queue depth, concurrency, or scale events, because prefill time itself doesn't. + +Short-prompt bias usually cancels: the scorer only ranks, so a shared bias is harmless. + **P50 and inflightAtP50** — current operating point (short window, default 3m): ``` @@ -45,15 +48,14 @@ The scorer draws a straight line through two points it has actually observed: - when there is no queue (`inflight = 0`): TTFT = P10Low (just the raw prefill time) - at the recent median load (`inflight = inflightAtP50`): TTFT = P50 -It then reads off that line at the current inflight to predict what the next request -will wait. +It then reads off that line at the current inflight to predict the next request's TTFT. ## Parameters | Parameter | Default | Description | |---|---|---| | `windowAge` | 3m | Window for P50 (short -- keeps P50 fresh and responsive) | -| `lowLoadWindowAge` | 1h | Window for two-level P10Low (long -- stable hardware floor) | +| `lowLoadWindowAge` | 1h | Window for two-level P10Low | | `windowSize` | 5000 | Ring buffer capacity (~200 KB per model) | | `minObservations` | 3 | Minimum observations required to compute any percentile | @@ -69,3 +71,16 @@ overloads, all traffic switches to the other, the first model drains and wins ag P(model i) proportional to score_i^(1/T) # T = temperature, default 1.0 ``` At T = 1.0, a model scoring 0.8 vs 0.2 receives ~80% vs 20% of requests. + + +### Prompt-length-aware floor + +P10Low is estimated from the fastest observed completions, which tend to be short-prompt +requests. For a long-prompt request, the prefill time is intrinsically higher, so the scorer under-predicts TTFT even at zero queue depth. + +A more accurate floor would scale with the incoming prompt token count: +``` +P10Low(tokens) = base_prefill + tokens × prefill_rate +``` +where `base_prefill` and `prefill_rate` are fit from observations bucketed by prompt length. +This matters most when the workload has high prompt-length variance. From 8eaa3cae87c00f0b297252c7372581518995acfa Mon Sep 17 00:00:00 2001 From: Mohammad Date: Mon, 29 Jun 2026 10:33:20 +0300 Subject: [PATCH 08/27] Add dynamic window for P50. Signed-off-by: Mohammad --- examples/medianttft-values.yaml | 13 +- .../datalayer/ttftpercentile/plugin.go | 141 ++++++++----- .../datalayer/ttftpercentile/tracker.go | 50 +++-- .../modelselector/scorer/queuettft/plugin.go | 194 ++++++++++++++---- 4 files changed, 282 insertions(+), 116 deletions(-) diff --git a/examples/medianttft-values.yaml b/examples/medianttft-values.yaml index 53c9aa46..89974b9f 100644 --- a/examples/medianttft-values.yaml +++ b/examples/medianttft-values.yaml @@ -11,15 +11,20 @@ payloadProcessor: - type: base-model-to-header - type: model-selector - type: queue-ttft-scorer + # effectiveTTFT = P10Low + inflight x (P50 - P10Low) / inflightAtP50 + # line through (0, P10Low) and (inflightAtP50, P50), extrapolated linearly + parameters: + explorationRate: 0.1 # 10% of requests probe under-observed models; 0 = disabled - type: max-score-picker - type: ttft-percentile-extractor parameters: intervalDuration: 1s windowSize: 5000 - windowAge: 1m - minObservations: 3 + maxObservationAge: 3m inflightEmaAlpha: 0.2 - lowLoadWindowAge: 1h # window for two-level P10Low (long = stable hardware floor) + lowLoadWindowAge: 1h + maxRequests: 100 + minRequests: 20 - type: model-config-datasource parameters: modelsPath: /config/models.json @@ -28,7 +33,7 @@ payloadProcessor: plugins: request: - pluginRef: model-selector - - pluginRef: median-ttft-scorer + - pluginRef: queue-ttft-scorer weight: 1.0 - pluginRef: max-score-picker - pluginRef: body-field-to-header diff --git a/pkg/framework/plugins/datalayer/ttftpercentile/plugin.go b/pkg/framework/plugins/datalayer/ttftpercentile/plugin.go index cb365c7b..ca2408fc 100644 --- a/pkg/framework/plugins/datalayer/ttftpercentile/plugin.go +++ b/pkg/framework/plugins/datalayer/ttftpercentile/plugin.go @@ -15,7 +15,7 @@ limitations under the License. */ // Package ttftpercentile tracks per-model TTFT distributions and publishes -// P10Low, P50, and inflightAtP50 for the median-ttft-scorer. +// P10Low, P50, and inflightAtP50 for the queue-ttft-scorer. package ttftpercentile import ( @@ -37,11 +37,12 @@ const ( AttributeKey = "ttft-percentile" defaultWindowSize = 5000 - defaultWindowAge = 1 * time.Minute - defaultMinObservations = 3 + defaultMaxObservationAge = 3 * time.Minute // observations older than this are never used defaultIntervalDuration = 5 * time.Second defaultInflightEMAAlpha = 0.2 defaultLowLoadWindowAge = 1 * time.Hour + defaultMaxRequests = 100 // cap the short window to the most recent N observations + defaultMinRequests = 10 // below this count the scorer falls back to the optimistic seed ) var _ dlsrc.Extractor = &TTFTPercentileExtractor{} @@ -49,21 +50,29 @@ var _ dlsrc.Extractor = &TTFTPercentileExtractor{} type TTFTPercentileExtractorConfig struct { IntervalDuration string `json:"intervalDuration,omitempty"` WindowSize int `json:"windowSize,omitempty"` - WindowAge string `json:"windowAge,omitempty"` - MinObservations int `json:"minObservations,omitempty"` - InflightEMAAlpha float64 `json:"inflightEmaAlpha,omitempty"` - LowLoadWindowAge string `json:"lowLoadWindowAge,omitempty"` + // MaxObservationAge caps how far back the short window looks. + // Observations older than this are never used for P50 or short-window P10. + MaxObservationAge string `json:"maxObservationAge,omitempty"` + InflightEMAAlpha float64 `json:"inflightEmaAlpha,omitempty"` + LowLoadWindowAge string `json:"lowLoadWindowAge,omitempty"` + // MaxRequests caps the short window to the most recent N observations regardless of age. + MaxRequests int `json:"maxRequests,omitempty"` + // MinRequests is the minimum capped-window count for the scorer to use the trusted + // operating point. Below this the scorer falls back to the optimistic seed (floor only). + MinRequests int `json:"minRequests,omitempty"` } // TTFTPercentileMetrics is written to each model's attribute store every intervalDuration. type TTFTPercentileMetrics struct { - Requests int64 - AvgInflight float64 - InflightAtP50 float64 // avg inflight_at_dispatch of observations in the P40-P60 band - P10LowTTFT float64 // two-level P10: P10 of the bottom decile; hardware-floor estimate - P10TTFT float64 // P10 from all obs (short window) - P50TTFT float64 // P50 from all obs (short window) + Requests int64 + AvgInflight float64 + InflightAtP50 float64 // avg inflight_at_dispatch of observations in the P40-P60 band + P10LowTTFT float64 // two-level P10: P10 of the bottom decile; hardware-floor estimate + P10TTFT float64 // P10 from capped short window + P50TTFT float64 // P50 from capped short window LastObservedAt int64 + RecentN int // count of observations in the capped short window + MinRequests int // scorer threshold — copied from config so the scorer needs no separate param } func (m TTFTPercentileMetrics) Clone() datalayer.Cloneable { return m } @@ -81,39 +90,44 @@ type modelPercentileState struct { avgInflightInit bool } -func (s *modelPercentileState) flush(now time.Time, windowAge, lowLoadWindowAge time.Duration, minObs int) { - p50, inflightAtP50, ok50 := s.tracker.quantileWithInflight(0.50, now, windowAge, minObs) - p10, ok10 := s.tracker.quantile(0.10, now, windowAge, minObs) +func (s *modelPercentileState) flush(now time.Time, maxObservationAge, lowLoadWindowAge time.Duration, maxRequests int) { + p50, inflightAtP50, ok50 := s.tracker.quantileWithInflight(0.50, now, maxObservationAge, maxRequests) + p10, ok10 := s.tracker.quantile(0.10, now, maxObservationAge, maxRequests) if ok50 { s.P50TTFT, s.InflightAtP50, s.LastObservedAt = p50, inflightAtP50, now.UnixNano() } if ok10 { s.P10TTFT = p10 } - if p10low, ok := s.tracker.p10Low(now, lowLoadWindowAge, minObs); ok { + if p10low, ok := s.tracker.p10Low(now, lowLoadWindowAge); ok { s.P10LowTTFT = p10low } + s.RecentN = s.tracker.countCapped(now, maxObservationAge, maxRequests) s.intervalStart = now } type TTFTPercentileExtractor struct { - typedName plugin.TypedName - ds datalayer.Datastore - state map[string]*modelPercentileState - windowSize int - windowAge time.Duration - minObservations int - intervalDuration time.Duration - inflightEMAAlpha float64 - lowLoadWindowAge time.Duration + typedName plugin.TypedName + ds datalayer.Datastore + state map[string]*modelPercentileState + windowSize int + maxObservationAge time.Duration + intervalDuration time.Duration + inflightEMAAlpha float64 + lowLoadWindowAge time.Duration + maxRequests int + minRequests int } func ExtractorFactory(name string, parameters json.RawMessage, h plugin.Handle) (plugin.Plugin, error) { cfg := TTFTPercentileExtractorConfig{ - IntervalDuration: defaultIntervalDuration.String(), WindowSize: defaultWindowSize, - WindowAge: defaultWindowAge.String(), MinObservations: defaultMinObservations, - InflightEMAAlpha: defaultInflightEMAAlpha, - LowLoadWindowAge: defaultLowLoadWindowAge.String(), + IntervalDuration: defaultIntervalDuration.String(), + WindowSize: defaultWindowSize, + MaxObservationAge: defaultMaxObservationAge.String(), + InflightEMAAlpha: defaultInflightEMAAlpha, + LowLoadWindowAge: defaultLowLoadWindowAge.String(), + MaxRequests: defaultMaxRequests, + MinRequests: defaultMinRequests, } if len(parameters) > 0 { if err := json.Unmarshal(parameters, &cfg); err != nil { @@ -123,8 +137,11 @@ func ExtractorFactory(name string, parameters json.RawMessage, h plugin.Handle) if cfg.WindowSize <= 0 { return nil, fmt.Errorf("windowSize must be > 0 for plugin %q", name) } - if cfg.MinObservations <= 0 { - return nil, fmt.Errorf("minObservations must be > 0 for plugin %q", name) + if cfg.MaxRequests <= 0 { + return nil, fmt.Errorf("maxRequests must be > 0 for plugin %q", name) + } + if cfg.MinRequests <= 0 { + return nil, fmt.Errorf("minRequests must be > 0 for plugin %q", name) } if cfg.InflightEMAAlpha <= 0 || cfg.InflightEMAAlpha > 1 { return nil, fmt.Errorf("inflightEmaAlpha must be in (0,1] for plugin %q", name) @@ -133,28 +150,35 @@ func ExtractorFactory(name string, parameters json.RawMessage, h plugin.Handle) if err != nil { return nil, fmt.Errorf("invalid intervalDuration %q for plugin %q: %w", cfg.IntervalDuration, name, err) } - windowAge, err := time.ParseDuration(cfg.WindowAge) + maxObsAge, err := time.ParseDuration(cfg.MaxObservationAge) if err != nil { - return nil, fmt.Errorf("invalid windowAge %q for plugin %q: %w", cfg.WindowAge, name, err) + return nil, fmt.Errorf("invalid maxObservationAge %q for plugin %q: %w", cfg.MaxObservationAge, name, err) } lowLoadAge, err := time.ParseDuration(cfg.LowLoadWindowAge) if err != nil { return nil, fmt.Errorf("invalid lowLoadWindowAge %q for plugin %q: %w", cfg.LowLoadWindowAge, name, err) } return NewTTFTPercentileExtractor(h.Datastore()). - WithName(name).WithIntervalDuration(interval). - WithWindow(cfg.WindowSize, windowAge, cfg.MinObservations). + WithName(name). + WithIntervalDuration(interval). + WithWindow(cfg.WindowSize, maxObsAge). WithInflightEMAAlpha(cfg.InflightEMAAlpha). - WithLowLoadWindowAge(lowLoadAge), nil + WithLowLoadWindowAge(lowLoadAge). + WithRequestBounds(cfg.MaxRequests, cfg.MinRequests), nil } func NewTTFTPercentileExtractor(ds datalayer.Datastore) *TTFTPercentileExtractor { return &TTFTPercentileExtractor{ - typedName: plugin.TypedName{Type: PluginType, Name: PluginType}, - ds: ds, state: make(map[string]*modelPercentileState), - windowSize: defaultWindowSize, windowAge: defaultWindowAge, - minObservations: defaultMinObservations, intervalDuration: defaultIntervalDuration, - inflightEMAAlpha: defaultInflightEMAAlpha, lowLoadWindowAge: defaultLowLoadWindowAge, + typedName: plugin.TypedName{Type: PluginType, Name: PluginType}, + ds: ds, + state: make(map[string]*modelPercentileState), + windowSize: defaultWindowSize, + maxObservationAge: defaultMaxObservationAge, + intervalDuration: defaultIntervalDuration, + inflightEMAAlpha: defaultInflightEMAAlpha, + lowLoadWindowAge: defaultLowLoadWindowAge, + maxRequests: defaultMaxRequests, + minRequests: defaultMinRequests, } } @@ -165,8 +189,8 @@ func (e *TTFTPercentileExtractor) WithName(n string) *TTFTPercentileExtractor { func (e *TTFTPercentileExtractor) WithIntervalDuration(d time.Duration) *TTFTPercentileExtractor { e.intervalDuration = d; return e } -func (e *TTFTPercentileExtractor) WithWindow(size int, age time.Duration, minObs int) *TTFTPercentileExtractor { - e.windowSize, e.windowAge, e.minObservations = size, age, minObs; return e +func (e *TTFTPercentileExtractor) WithWindow(size int, maxObsAge time.Duration) *TTFTPercentileExtractor { + e.windowSize, e.maxObservationAge = size, maxObsAge; return e } func (e *TTFTPercentileExtractor) WithInflightEMAAlpha(a float64) *TTFTPercentileExtractor { e.inflightEMAAlpha = a; return e @@ -174,6 +198,9 @@ func (e *TTFTPercentileExtractor) WithInflightEMAAlpha(a float64) *TTFTPercentil func (e *TTFTPercentileExtractor) WithLowLoadWindowAge(age time.Duration) *TTFTPercentileExtractor { e.lowLoadWindowAge = age; return e } +func (e *TTFTPercentileExtractor) WithRequestBounds(maxN, minN int) *TTFTPercentileExtractor { + e.maxRequests, e.minRequests = maxN, minN; return e +} func (e *TTFTPercentileExtractor) Extract(ctx context.Context, events []dlsrc.Event) error { debugLogger := log.FromContext(ctx).V(logutil.DEBUG) @@ -237,7 +264,7 @@ func (e *TTFTPercentileExtractor) Extract(ctx context.Context, events []dlsrc.Ev } else { s.AvgInflight = e.inflightEMAAlpha*sample + (1-e.inflightEMAAlpha)*s.AvgInflight } - s.flush(now, e.windowAge, e.lowLoadWindowAge, e.minObservations) + s.flush(now, e.maxObservationAge, e.lowLoadWindowAge, e.maxRequests) } updated[model] = true } @@ -245,29 +272,35 @@ func (e *TTFTPercentileExtractor) Extract(ctx context.Context, events []dlsrc.Ev for model := range updated { s := e.state[model] - cutoff := now.Add(-e.windowAge) + cutoff := now.Add(-e.maxObservationAge) for id, entry := range s.pending { if entry.dispatchedAt.Before(cutoff) { delete(s.pending, id) } } m := s.TTFTPercentileMetrics + m.MinRequests = e.minRequests e.ds.GetOrCreateModel(model).GetAttributes().Put(AttributeKey, m) - p10eff := m.P10LowTTFT - if p10eff == 0 { - p10eff = m.P10TTFT + + // Compute effectiveTTFT for the debug log (mirrors scorer logic). + floor := m.P10LowTTFT + if floor == 0 { + floor = m.P10TTFT } - eff := p10eff - if p10eff > 0 && m.InflightAtP50 > 0 && m.P50TTFT > p10eff { - eff = p10eff + float64(m.Requests)*(m.P50TTFT-p10eff)/m.InflightAtP50 - if eff < p10eff { - eff = p10eff + var eff float64 + if floor > 0 && m.RecentN >= m.MinRequests && m.InflightAtP50 > 0 && m.P50TTFT > floor { + eff = floor + float64(m.Requests)*(m.P50TTFT-floor)/m.InflightAtP50 + if eff < floor { + eff = floor } + } else { + eff = floor // optimistic seed or unobserved } debugLogger.Info("ttft-percentile wrote attribute", "model", model, "Requests", m.Requests, "AvgInflight", m.AvgInflight, "InflightAtP50", m.InflightAtP50, "P10Low_s", m.P10LowTTFT, "P10_s", m.P10TTFT, "P50_s", m.P50TTFT, "EffectiveTTFT_s", eff, + "RecentN", m.RecentN, "MinRequests", m.MinRequests, ) } return nil diff --git a/pkg/framework/plugins/datalayer/ttftpercentile/tracker.go b/pkg/framework/plugins/datalayer/ttftpercentile/tracker.go index 2be599f0..85b8f93f 100644 --- a/pkg/framework/plugins/datalayer/ttftpercentile/tracker.go +++ b/pkg/framework/plugins/datalayer/ttftpercentile/tracker.go @@ -23,15 +23,19 @@ import ( type percentileTracker interface { add(value float64, inflight int64, ts time.Time) - quantile(p float64, now time.Time, maxAge time.Duration, minCount int) (float64, bool) + // quantile returns the p-th percentile of the most recent min(maxN, available) + // observations within maxAge. Returns false if there are no observations. + quantile(p float64, now time.Time, maxAge time.Duration, maxN int) (float64, bool) // quantileWithInflight returns the p-th percentile TTFT and the average inflight - // of observations in the [p-0.1, p+0.1] band — more stable than a single-point inflight. - quantileWithInflight(p float64, now time.Time, maxAge time.Duration, minCount int) (float64, float64, bool) - // p10Low computes a two-level P10: first finds the P10 TTFT threshold from all - // observations, then returns the P10 of observations at or below that threshold. - // This isolates low-queue-wait observations without requiring a fixed inflight cut-off, - // so P10Low updates continuously regardless of sustained load level. - p10Low(now time.Time, maxAge time.Duration, minCount int) (float64, bool) + // of observations in the [p-0.1, p+0.1] band of the capped short window. + // Returns false if there are no observations. + quantileWithInflight(p float64, now time.Time, maxAge time.Duration, maxN int) (float64, float64, bool) + // p10Low computes the two-level P10 over the full lowLoadWindowAge window (no cap). + // It isolates low-queue-wait observations without requiring a fixed inflight cut-off. + // Returns false if there are no observations. + p10Low(now time.Time, maxAge time.Duration) (float64, bool) + // countCapped returns the number of observations in the capped short window. + countCapped(now time.Time, maxAge time.Duration, maxN int) int } type observation struct { @@ -58,6 +62,7 @@ func (t *slidingWindowTracker) add(value float64, inflight int64, ts time.Time) } } +// recentObs returns observations within maxAge, newest first. func (t *slidingWindowTracker) recentObs(now time.Time, maxAge time.Duration) []observation { cutoff := now.Add(-maxAge) cap := len(t.buf) @@ -71,9 +76,23 @@ func (t *slidingWindowTracker) recentObs(now time.Time, maxAge time.Duration) [] return out } -func (t *slidingWindowTracker) quantile(p float64, now time.Time, maxAge time.Duration, minCount int) (float64, bool) { +// recentObsCapped returns the most recent min(maxN, available) observations within maxAge. +// recentObs is already newest-first so a prefix slice gives the most recent maxN. +func (t *slidingWindowTracker) recentObsCapped(now time.Time, maxAge time.Duration, maxN int) []observation { obs := t.recentObs(now, maxAge) - if len(obs) < minCount { + if maxN > 0 && len(obs) > maxN { + obs = obs[:maxN] + } + return obs +} + +func (t *slidingWindowTracker) countCapped(now time.Time, maxAge time.Duration, maxN int) int { + return len(t.recentObsCapped(now, maxAge, maxN)) +} + +func (t *slidingWindowTracker) quantile(p float64, now time.Time, maxAge time.Duration, maxN int) (float64, bool) { + obs := t.recentObsCapped(now, maxAge, maxN) + if len(obs) == 0 { return 0, false } vals := make([]float64, len(obs)) @@ -89,9 +108,10 @@ func (t *slidingWindowTracker) quantile(p float64, now time.Time, maxAge time.Du return vals[lo] + (idx-float64(lo))*(vals[lo+1]-vals[lo]), true } -func (t *slidingWindowTracker) p10Low(now time.Time, maxAge time.Duration, minCount int) (float64, bool) { +func (t *slidingWindowTracker) p10Low(now time.Time, maxAge time.Duration) (float64, bool) { + // Uses the full lowLoadWindowAge window with no cap — stable hardware floor. obs := t.recentObs(now, maxAge) - if len(obs) < minCount { + if len(obs) == 0 { return 0, false } vals := make([]float64, len(obs)) @@ -115,9 +135,9 @@ func (t *slidingWindowTracker) p10Low(now time.Time, maxAge time.Duration, minCo return low[lo2] + (idx2-float64(lo2))*(low[lo2+1]-low[lo2]), true } -func (t *slidingWindowTracker) quantileWithInflight(p float64, now time.Time, maxAge time.Duration, minCount int) (float64, float64, bool) { - obs := t.recentObs(now, maxAge) - if len(obs) < minCount { +func (t *slidingWindowTracker) quantileWithInflight(p float64, now time.Time, maxAge time.Duration, maxN int) (float64, float64, bool) { + obs := t.recentObsCapped(now, maxAge, maxN) + if len(obs) == 0 { return 0, 0, false } sort.Slice(obs, func(i, j int) bool { return obs[i].value < obs[j].value }) diff --git a/pkg/framework/plugins/modelselector/scorer/queuettft/plugin.go b/pkg/framework/plugins/modelselector/scorer/queuettft/plugin.go index b579b804..173cde50 100644 --- a/pkg/framework/plugins/modelselector/scorer/queuettft/plugin.go +++ b/pkg/framework/plugins/modelselector/scorer/queuettft/plugin.go @@ -17,12 +17,16 @@ limitations under the License. // Package queuettft scores models by predicted TTFT under current load. // effectiveTTFT = P10Low + inflight × (P50 − P10Low) / inflightAtP50: // a line through (0, P10Low) and (inflightAtP50, P50), extrapolated linearly. +// Under-observed models receive an optimistic seed (their own floor) so they +// keep competing for traffic instead of stalling at a fixed 0.5 score. package queuettft import ( "context" "encoding/json" + "fmt" "math" + "math/rand" "sigs.k8s.io/controller-runtime/pkg/log" @@ -34,21 +38,49 @@ import ( "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/plugins/datalayer/ttftpercentile" ) -const PluginType = "queue-ttft-scorer" +const ( + PluginType = "queue-ttft-scorer" + // unobserved is a sentinel returned by effectiveTTFT when no floor exists yet. + // It is negative so it cannot be confused with a valid (non-negative) TTFT estimate. + unobserved = -1.0 + + defaultExplorationRate = 0.0 // off by default; set e.g. 0.1 for 10% exploration +) var _ modelselector.Scorer = &QueueTTFTScorer{} +// QueueTTFTScorerConfig holds optional parameters for the scorer plugin. +type QueueTTFTScorerConfig struct { + // ExplorationRate controls the probability of routing a request to an under-observed + // model (UNOBSERVED or SEED state) instead of the trusted best model. A value of 0.1 + // means ~10% of requests probe the under-observed model, preventing a burst of traffic + // before the first responses return and P50 is calibrated. + // Range [0, 1]. Default 0 (disabled — every request goes to the winner). + ExplorationRate float64 `json:"explorationRate,omitempty"` +} + type QueueTTFTScorer struct { - typedName plugin.TypedName + typedName plugin.TypedName + explorationRate float64 } -func ScorerFactory(name string, _ json.RawMessage, _ plugin.Handle) (plugin.Plugin, error) { - return NewQueueTTFTScorer().WithName(name), nil +func ScorerFactory(name string, parameters json.RawMessage, _ plugin.Handle) (plugin.Plugin, error) { + cfg := QueueTTFTScorerConfig{ExplorationRate: defaultExplorationRate} + if len(parameters) > 0 { + if err := json.Unmarshal(parameters, &cfg); err != nil { + return nil, fmt.Errorf("failed to parse parameters for plugin %q: %w", name, err) + } + } + if cfg.ExplorationRate < 0 || cfg.ExplorationRate > 1 { + return nil, fmt.Errorf("explorationRate must be in [0, 1] for plugin %q", name) + } + return NewQueueTTFTScorer().WithName(name).WithExplorationRate(cfg.ExplorationRate), nil } func NewQueueTTFTScorer() *QueueTTFTScorer { return &QueueTTFTScorer{ - typedName: plugin.TypedName{Type: PluginType, Name: PluginType}, + typedName: plugin.TypedName{Type: PluginType, Name: PluginType}, + explorationRate: defaultExplorationRate, } } @@ -56,81 +88,157 @@ func (s *QueueTTFTScorer) TypedName() plugin.TypedName { return s.typedName } func (s *QueueTTFTScorer) WithName(name string) *QueueTTFTScorer { s.typedName.Name = name; return s } +func (s *QueueTTFTScorer) WithExplorationRate(r float64) *QueueTTFTScorer { + s.explorationRate = r; return s +} // Score returns (maxTTFT − effectiveTTFT) / (maxTTFT − minTTFT) per model. -// Unobserved models score 1.0 (all unobserved) or 0.5 (some peers observed). +// +// Three cases per model: +// 1. Truly cold (no floor): unobserved sentinel → seeded to min(observed) after all are computed. +// 2. Trusted: RecentN ≥ MinRequests with a calibrated P50 → formula score. +// 3. Gone-quiet / under-observed: floor only (optimistic, assumes no queue). +// +// If all models are cold → all score 1.0. +// +// If explorationRate > 0, models in state 1 or 3 are suppressed to score 0 with probability +// (1 - explorationRate). This throttles probing traffic to ~explorationRate of requests, +// preventing a burst of traffic to under-observed models before their first responses return. func (s *QueueTTFTScorer) Score(ctx context.Context, _ *plugin.CycleState, _ *requesthandling.InferenceRequest, models []datalayer.Model) map[datalayer.Model]float64 { - ttfts := make(map[datalayer.Model]float64, len(models)) - minTTFT, maxTTFT := math.MaxFloat64, 0.0 + effs := make(map[datalayer.Model]float64, len(models)) + needsProbe := make(map[datalayer.Model]bool, len(models)) + minEff, maxEff := math.MaxFloat64, 0.0 allUnobserved := true for _, model := range models { - v := s.effectiveTTFT(ctx, model) - ttfts[model] = v - if v > 0 { + v, probe := s.effectiveTTFT(ctx, model) + effs[model] = v + needsProbe[model] = probe + if v != unobserved { allUnobserved = false - if v > maxTTFT { - maxTTFT = v + if v > maxEff { + maxEff = v } - if v < minTTFT { - minTTFT = v + if v < minEff { + minEff = v } } } + // All models cold → explore equally (no suppression). + if allUnobserved { + scores := make(map[datalayer.Model]float64, len(models)) + for _, model := range models { + scores[model] = 1.0 + } + return scores + } + + // Seed truly cold models at the best observed TTFT (optimistic: assume as-good-as-best). + seed := minEff + for _, model := range models { + if effs[model] == unobserved { + effs[model] = seed + log.FromContext(ctx).V(logutil.DEBUG).Info("queue-ttft cold-seed", + "model", model.GetName(), "seed_s", seed, + ) + } + } + + // Recompute range after seeding (seed == minEff so max is unchanged, but be explicit). + minEff, maxEff = math.MaxFloat64, 0.0 + for _, v := range effs { + if v > maxEff { + maxEff = v + } + if v < minEff { + minEff = v + } + } + scores := make(map[datalayer.Model]float64, len(models)) for _, model := range models { - v := ttfts[model] - switch { - case v == 0 && allUnobserved: + if maxEff == minEff { scores[model] = 1.0 - case v == 0: - scores[model] = 0.5 - case maxTTFT == minTTFT: - scores[model] = 1.0 - default: - scores[model] = (maxTTFT - v) / (maxTTFT - minTTFT) + } else { + scores[model] = (maxEff - effs[model]) / (maxEff - minEff) + } + } + + // Exploration gate: under-observed models win only explorationRate fraction of requests. + // On the other (1-explorationRate) fraction, suppress their score to 0 so the best + // calibrated model wins and the under-observed model does not receive a traffic burst. + if s.explorationRate > 0 { + for _, model := range models { + if needsProbe[model] && rand.Float64() >= s.explorationRate { + scores[model] = 0 + log.FromContext(ctx).V(logutil.DEBUG).Info("queue-ttft exploration suppressed", + "model", model.GetName(), "explorationRate", s.explorationRate, + ) + } } } if dl := log.FromContext(ctx).V(logutil.DEBUG); dl.Enabled() { for _, model := range models { - dl.Info("queue-ttft score", "model", model.GetName(), "effectiveTTFT", ttfts[model], "score", scores[model]) + dl.Info("queue-ttft score", + "model", model.GetName(), + "effectiveTTFT", effs[model], + "score", scores[model], + "needsProbe", needsProbe[model], + ) } } return scores } -func (s *QueueTTFTScorer) effectiveTTFT(ctx context.Context, model datalayer.Model) float64 { +// effectiveTTFT returns the predicted TTFT for the model's next request and whether +// the model needs exploration (UNOBSERVED or SEED state — not yet calibrated). +// +// - (unobserved, true): no floor yet — truly cold model. +// - (floor, true): under-observed or gone-quiet — optimistic seed, assume no queue. +// - (floor + inflight×slope, false): trusted operating point with recent calibration. +func (s *QueueTTFTScorer) effectiveTTFT(ctx context.Context, model datalayer.Model) (float64, bool) { val, ok := model.GetAttributes().Get(ttftpercentile.AttributeKey) if !ok { - return 0 + return unobserved, true } m, ok := val.(ttftpercentile.TTFTPercentileMetrics) if !ok { - return 0 + return unobserved, true } - p10 := m.P10LowTTFT - if p10 == 0 { - p10 = m.P10TTFT + + floor := m.P10LowTTFT + if floor == 0 { + floor = m.P10TTFT } - if p10 == 0 { - return 0 + if floor == 0 { + return unobserved, true // truly cold: no hardware floor established yet } - // effectiveTTFT = P10Low + inflight × (P50 − P10Low) / inflightAtP50 - // Falls back to P10Low when P50 is not yet available or equals the floor. - eff := p10 - if m.InflightAtP50 > 0 && m.P50TTFT > p10 { - eff = p10 + float64(m.Requests)*(m.P50TTFT-p10)/m.InflightAtP50 + // Trusted: enough recent observations and a calibrated operating point. + if m.RecentN >= m.MinRequests && m.InflightAtP50 > 0 && m.P50TTFT > floor { + eff := floor + float64(m.Requests)*(m.P50TTFT-floor)/m.InflightAtP50 + if eff < floor { + eff = floor + } + if dl := log.FromContext(ctx).V(logutil.DEBUG); dl.Enabled() { + dl.Info("queue-ttft effective trusted", + "model", model.GetName(), "inflight", m.Requests, + "inflightAtP50", m.InflightAtP50, "P10Low_s", m.P10LowTTFT, + "P50_s", m.P50TTFT, "recentN", m.RecentN, "effectiveTTFT", eff, + ) + } + return eff, false } + // Optimistic seed: model is under-observed or gone-quiet. + // Assume no queue — predict only the hardware floor. if dl := log.FromContext(ctx).V(logutil.DEBUG); dl.Enabled() { - dl.Info("queue-ttft effective", - "model", model.GetName(), "inflight", m.Requests, - "inflightAtP50", m.InflightAtP50, "P10Low_s", m.P10LowTTFT, - "P10_s", m.P10TTFT, "P50_s", m.P50TTFT, "effectiveTTFT", eff, + dl.Info("queue-ttft effective seed", + "model", model.GetName(), "floor_s", floor, + "recentN", m.RecentN, "minRequests", m.MinRequests, ) } - return eff + return floor, true } From 7426c0e9586a5718e81657bcf9967d03aa339f09 Mon Sep 17 00:00:00 2001 From: Mohammad Date: Tue, 30 Jun 2026 18:17:48 +0300 Subject: [PATCH 09/27] Add denug headers plugin. Signed-off-by: Mohammad --- cmd/runner/runner.go | 2 + examples/medianttft-values.yaml | 7 +- .../interface/requesthandling/types.go | 3 + .../modelselector/scorer/queuettft/plugin.go | 76 +++++++++++- .../requesthandling/modelselector/plugin.go | 4 + .../scoringdebugheaders/plugin.go | 111 ++++++++++++++++++ pkg/handlers/response.go | 6 +- 7 files changed, 200 insertions(+), 9 deletions(-) create mode 100644 pkg/framework/plugins/responsehandling/scoringdebugheaders/plugin.go diff --git a/cmd/runner/runner.go b/cmd/runner/runner.go index 1563a7af..5d7e9697 100644 --- a/cmd/runner/runner.go +++ b/cmd/runner/runner.go @@ -56,6 +56,7 @@ import ( inflightrequestsscorer "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/plugins/modelselector/scorer/inflightrequests" queuettftscorer "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/plugins/modelselector/scorer/queuettft" "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/plugins/requesthandling/basemodelextractor" + scoringdebugheaders "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/plugins/responsehandling/scoringdebugheaders" "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/plugins/requesthandling/bodyfieldtoheader" modelselectorplugin "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/plugins/requesthandling/modelselector" "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/plugins/requesthandling/profilepicker/single" @@ -289,6 +290,7 @@ func (r *Runner) registerInTreePlugins() { plugin.Register(modelselectorplugin.ModelSelectorPluginType, modelselectorplugin.ModelSelectorPluginFactory) plugin.Register(inflightrequestsscorer.PluginType, inflightrequestsscorer.ScorerFactory) plugin.Register(queuettftscorer.PluginType, queuettftscorer.ScorerFactory) + plugin.Register(scoringdebugheaders.PluginType, scoringdebugheaders.Factory) } // registerHealthServer adds the Health gRPC server as a Runnable to the given manager. diff --git a/examples/medianttft-values.yaml b/examples/medianttft-values.yaml index 89974b9f..f3a8ad97 100644 --- a/examples/medianttft-values.yaml +++ b/examples/medianttft-values.yaml @@ -1,7 +1,7 @@ payloadProcessor: listModels: - - facebook/opt-125m - - facebook/opt-350m + - Qwen/Qwen3-8B + - Qwen/Qwen3-32B customConfig: plugins: - type: body-field-to-header @@ -25,6 +25,7 @@ payloadProcessor: lowLoadWindowAge: 1h maxRequests: 100 minRequests: 20 + - type: scoring-debug-headers - type: model-config-datasource parameters: modelsPath: /config/models.json @@ -38,6 +39,8 @@ payloadProcessor: - pluginRef: max-score-picker - pluginRef: body-field-to-header - pluginRef: base-model-to-header + response: + - pluginRef: scoring-debug-headers datalayer: extractors: - pluginRef: ttft-percentile-extractor diff --git a/pkg/framework/interface/requesthandling/types.go b/pkg/framework/interface/requesthandling/types.go index 78afa58d..c79a7a77 100644 --- a/pkg/framework/interface/requesthandling/types.go +++ b/pkg/framework/interface/requesthandling/types.go @@ -22,6 +22,9 @@ import ( "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/interface/plugin" ) +// TTFTCycleStateKey holds the real TTFT (seconds, float64) measured by the handler. +const TTFTCycleStateKey = "ipp/ttft-seconds" + func newInferenceMessage() InferenceMessage { return InferenceMessage{ Headers: map[string]string{}, diff --git a/pkg/framework/plugins/modelselector/scorer/queuettft/plugin.go b/pkg/framework/plugins/modelselector/scorer/queuettft/plugin.go index 173cde50..d6c083db 100644 --- a/pkg/framework/plugins/modelselector/scorer/queuettft/plugin.go +++ b/pkg/framework/plugins/modelselector/scorer/queuettft/plugin.go @@ -41,12 +41,25 @@ import ( const ( PluginType = "queue-ttft-scorer" // unobserved is a sentinel returned by effectiveTTFT when no floor exists yet. - // It is negative so it cannot be confused with a valid (non-negative) TTFT estimate. unobserved = -1.0 defaultExplorationRate = 0.0 // off by default; set e.g. 0.1 for 10% exploration + + // DecisionsCycleStateKey holds the per-model scoring decisions for response plugins. + DecisionsCycleStateKey = "queue-ttft/decisions" ) +// ScoringDecision captures the scorer's decision for one model, written to CycleState. +type ScoringDecision struct { + EffectiveTTFT float64 + Floor float64 // P10Low + P50 float64 + RecentN int + Inflight int64 + State string // "trusted" | "seed" | "unobserved" + ExplorationSuppressed bool +} + var _ modelselector.Scorer = &QueueTTFTScorer{} // QueueTTFTScorerConfig holds optional parameters for the scorer plugin. @@ -104,9 +117,10 @@ func (s *QueueTTFTScorer) WithExplorationRate(r float64) *QueueTTFTScorer { // If explorationRate > 0, models in state 1 or 3 are suppressed to score 0 with probability // (1 - explorationRate). This throttles probing traffic to ~explorationRate of requests, // preventing a burst of traffic to under-observed models before their first responses return. -func (s *QueueTTFTScorer) Score(ctx context.Context, _ *plugin.CycleState, _ *requesthandling.InferenceRequest, models []datalayer.Model) map[datalayer.Model]float64 { +func (s *QueueTTFTScorer) Score(ctx context.Context, cycleState *plugin.CycleState, _ *requesthandling.InferenceRequest, models []datalayer.Model) map[datalayer.Model]float64 { effs := make(map[datalayer.Model]float64, len(models)) needsProbe := make(map[datalayer.Model]bool, len(models)) + wasUnobserved := make(map[datalayer.Model]bool, len(models)) minEff, maxEff := math.MaxFloat64, 0.0 allUnobserved := true @@ -114,7 +128,9 @@ func (s *QueueTTFTScorer) Score(ctx context.Context, _ *plugin.CycleState, _ *re v, probe := s.effectiveTTFT(ctx, model) effs[model] = v needsProbe[model] = probe - if v != unobserved { + if v == unobserved { + wasUnobserved[model] = true + } else { allUnobserved = false if v > maxEff { maxEff = v @@ -131,6 +147,7 @@ func (s *QueueTTFTScorer) Score(ctx context.Context, _ *plugin.CycleState, _ *re for _, model := range models { scores[model] = 1.0 } + s.writeDecisions(cycleState, models, effs, wasUnobserved, needsProbe, nil) return scores } @@ -165,13 +182,12 @@ func (s *QueueTTFTScorer) Score(ctx context.Context, _ *plugin.CycleState, _ *re } } - // Exploration gate: under-observed models win only explorationRate fraction of requests. - // On the other (1-explorationRate) fraction, suppress their score to 0 so the best - // calibrated model wins and the under-observed model does not receive a traffic burst. + explorationSuppressed := make(map[datalayer.Model]bool, len(models)) if s.explorationRate > 0 { for _, model := range models { if needsProbe[model] && rand.Float64() >= s.explorationRate { scores[model] = 0 + explorationSuppressed[model] = true log.FromContext(ctx).V(logutil.DEBUG).Info("queue-ttft exploration suppressed", "model", model.GetName(), "explorationRate", s.explorationRate, ) @@ -189,9 +205,57 @@ func (s *QueueTTFTScorer) Score(ctx context.Context, _ *plugin.CycleState, _ *re ) } } + + s.writeDecisions(cycleState, models, effs, wasUnobserved, needsProbe, explorationSuppressed) return scores } +// writeDecisions stores per-model scoring decisions in CycleState for response plugins. +func (s *QueueTTFTScorer) writeDecisions(cycleState *plugin.CycleState, models []datalayer.Model, + effs map[datalayer.Model]float64, wasUnobserved, needsProbe, explorationSuppressed map[datalayer.Model]bool) { + if cycleState == nil { + return + } + decisions := make(map[string]ScoringDecision, len(models)) + for _, model := range models { + var state string + switch { + case wasUnobserved[model]: + state = "unobserved" + case needsProbe[model]: + state = "seed" + default: + state = "trusted" + } + + var floor, p50 float64 + var recentN int + var inflight int64 + if val, ok := model.GetAttributes().Get(ttftpercentile.AttributeKey); ok { + if m, ok := val.(ttftpercentile.TTFTPercentileMetrics); ok { + floor = m.P10LowTTFT + if floor == 0 { + floor = m.P10TTFT + } + p50 = m.P50TTFT + recentN = m.RecentN + inflight = m.Requests + } + } + + decisions[model.GetName()] = ScoringDecision{ + EffectiveTTFT: effs[model], + Floor: floor, + P50: p50, + RecentN: recentN, + Inflight: inflight, + State: state, + ExplorationSuppressed: explorationSuppressed[model], + } + } + cycleState.Write(DecisionsCycleStateKey, decisions) +} + // effectiveTTFT returns the predicted TTFT for the model's next request and whether // the model needs exploration (UNOBSERVED or SEED state — not yet calibrated). // diff --git a/pkg/framework/plugins/requesthandling/modelselector/plugin.go b/pkg/framework/plugins/requesthandling/modelselector/plugin.go index 21168037..168d74e1 100644 --- a/pkg/framework/plugins/requesthandling/modelselector/plugin.go +++ b/pkg/framework/plugins/requesthandling/modelselector/plugin.go @@ -33,6 +33,9 @@ import ( const ( ModelSelectorPluginType = "model-selector" + + // SelectedModelCycleStateKey holds the selected model name for response plugins. + SelectedModelCycleStateKey = "model-selector/selected-model" ) var _ requesthandling.RequestProcessor = &ModelSelectorPlugin{} @@ -91,6 +94,7 @@ func (p *ModelSelectorPlugin) ProcessRequest(ctx context.Context, cycleState *pl selectedName := result.TargetModel.GetName() logger.V(logutil.VERBOSE).Info("Model selected", "model", selectedName) + cycleState.Write(SelectedModelCycleStateKey, selectedName) request.SetBodyField("model", selectedName) return nil diff --git a/pkg/framework/plugins/responsehandling/scoringdebugheaders/plugin.go b/pkg/framework/plugins/responsehandling/scoringdebugheaders/plugin.go new file mode 100644 index 00000000..be38bf23 --- /dev/null +++ b/pkg/framework/plugins/responsehandling/scoringdebugheaders/plugin.go @@ -0,0 +1,111 @@ +/* +Copyright 2026 The llm-d Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package scoringdebugheaders writes queue-ttft scoring decisions as response headers. +package scoringdebugheaders + +import ( + "context" + "encoding/json" + "fmt" + + "sigs.k8s.io/controller-runtime/pkg/log" + + logutil "github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/logging" + "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/interface/plugin" + "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/interface/requesthandling" + queuettft "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/plugins/modelselector/scorer/queuettft" + modelselector "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/plugins/requesthandling/modelselector" +) + +const ( + PluginType = "scoring-debug-headers" + + headerModel = "X-IPP-Scorer-Model" + headerState = "X-IPP-Scorer-State" + headerEffTTFT = "X-IPP-Scorer-EffectiveTTFT" + headerRealTTFT = "X-IPP-Real-TTFT" + headerFloor = "X-IPP-Scorer-Floor" + headerP50 = "X-IPP-Scorer-P50" + headerRecentN = "X-IPP-Scorer-RecentN" + headerInflight = "X-IPP-Scorer-Inflight" + headerExploration = "X-IPP-Scorer-ExplorationSuppressed" +) + +var _ requesthandling.ResponseProcessor = &ScoringDebugHeadersPlugin{} + +func Factory(name string, _ json.RawMessage, _ plugin.Handle) (plugin.Plugin, error) { + return NewScoringDebugHeadersPlugin().WithName(name), nil +} + +func NewScoringDebugHeadersPlugin() *ScoringDebugHeadersPlugin { + return &ScoringDebugHeadersPlugin{ + typedName: plugin.TypedName{Type: PluginType, Name: PluginType}, + } +} + +type ScoringDebugHeadersPlugin struct { + typedName plugin.TypedName +} + +func (p *ScoringDebugHeadersPlugin) TypedName() plugin.TypedName { return p.typedName } +func (p *ScoringDebugHeadersPlugin) WithName(name string) *ScoringDebugHeadersPlugin { + p.typedName.Name = name; return p +} + +func (p *ScoringDebugHeadersPlugin) ProcessResponse(ctx context.Context, cycleState *plugin.CycleState, response *requesthandling.InferenceResponse) error { + selectedModel, err := plugin.ReadCycleStateKey[string](cycleState, modelselector.SelectedModelCycleStateKey) + if err != nil { + log.FromContext(ctx).V(logutil.DEBUG).Info("scoring-debug-headers: no selected model in CycleState, skipping") + return nil + } + + decisions, err := plugin.ReadCycleStateKey[map[string]queuettft.ScoringDecision](cycleState, queuettft.DecisionsCycleStateKey) + if err != nil { + log.FromContext(ctx).V(logutil.DEBUG).Info("scoring-debug-headers: no scoring decisions in CycleState, skipping") + return nil + } + + d, ok := decisions[selectedModel] + if !ok { + log.FromContext(ctx).V(logutil.DEBUG).Info("scoring-debug-headers: selected model not in decisions", "model", selectedModel) + return nil + } + + response.SetHeader(headerModel, selectedModel) + response.SetHeader(headerState, d.State) + response.SetHeader(headerRecentN, fmt.Sprintf("%d", d.RecentN)) + response.SetHeader(headerInflight, fmt.Sprintf("%d", d.Inflight)) + + if d.Floor > 0 { + response.SetHeader(headerFloor, fmt.Sprintf("%.3f", d.Floor)) + } + if d.P50 > 0 { + response.SetHeader(headerP50, fmt.Sprintf("%.3f", d.P50)) + } + if d.EffectiveTTFT > 0 { + response.SetHeader(headerEffTTFT, fmt.Sprintf("%.3f", d.EffectiveTTFT)) + } + if d.ExplorationSuppressed { + response.SetHeader(headerExploration, "true") + } + + if realTTFT, err := plugin.ReadCycleStateKey[float64](cycleState, requesthandling.TTFTCycleStateKey); err == nil && realTTFT > 0 { + response.SetHeader(headerRealTTFT, fmt.Sprintf("%.3f", realTTFT)) + } + + return nil +} diff --git a/pkg/handlers/response.go b/pkg/handlers/response.go index 03210f11..6836577d 100644 --- a/pkg/handlers/response.go +++ b/pkg/handlers/response.go @@ -60,15 +60,19 @@ func (s *Server) HandleResponseHeaders(ctx context.Context, reqCtx *RequestConte // HandleResponseBody handles response bodies by executing response plugins in order. func (s *Server) HandleResponseBody(ctx context.Context, reqCtx *RequestContext, responseBodyBytes []byte) ([]*eppb.ProcessingResponse, error) { // Notify the data layer of the completed response. + ttft := reqCtx.ResponseFirstChunkTimestamp.Sub(reqCtx.RequestSentTimestamp) s.eventNotifier.Notify(datasource.Event{ Type: datasource.ResponseEventType, Payload: datasource.ResponsePayload{ Request: reqCtx.Request, Response: reqCtx.Response, Duration: reqCtx.ResponseCompleteTimestamp.Sub(reqCtx.RequestReceivedTimestamp), - TTFT: reqCtx.ResponseFirstChunkTimestamp.Sub(reqCtx.RequestSentTimestamp), + TTFT: ttft, }, }) + if ttft > 0 { + reqCtx.CycleState.Write(requesthandling.TTFTCycleStateKey, ttft.Seconds()) + } logger := log.FromContext(ctx) if len(reqCtx.Profile.ResponsePlugins) == 0 { From 613db1e4ce00fe72a2529fdd85f155ad63163538 Mon Sep 17 00:00:00 2001 From: Mohammad Date: Wed, 1 Jul 2026 19:19:19 +0300 Subject: [PATCH 10/27] Remove unused metric. Signed-off-by: Mohammad --- examples/medianttft-values.yaml | 1 - .../datalayer/ttftpercentile/plugin.go | 87 ++++--- .../modelselector/scorer/queuettft/README.md | 43 ++-- .../modelselector/scorer/queuettft/plugin.go | 221 ++++++------------ 4 files changed, 136 insertions(+), 216 deletions(-) diff --git a/examples/medianttft-values.yaml b/examples/medianttft-values.yaml index f3a8ad97..14b47844 100644 --- a/examples/medianttft-values.yaml +++ b/examples/medianttft-values.yaml @@ -21,7 +21,6 @@ payloadProcessor: intervalDuration: 1s windowSize: 5000 maxObservationAge: 3m - inflightEmaAlpha: 0.2 lowLoadWindowAge: 1h maxRequests: 100 minRequests: 20 diff --git a/pkg/framework/plugins/datalayer/ttftpercentile/plugin.go b/pkg/framework/plugins/datalayer/ttftpercentile/plugin.go index ca2408fc..64281807 100644 --- a/pkg/framework/plugins/datalayer/ttftpercentile/plugin.go +++ b/pkg/framework/plugins/datalayer/ttftpercentile/plugin.go @@ -39,7 +39,6 @@ const ( defaultWindowSize = 5000 defaultMaxObservationAge = 3 * time.Minute // observations older than this are never used defaultIntervalDuration = 5 * time.Second - defaultInflightEMAAlpha = 0.2 defaultLowLoadWindowAge = 1 * time.Hour defaultMaxRequests = 100 // cap the short window to the most recent N observations defaultMinRequests = 10 // below this count the scorer falls back to the optimistic seed @@ -52,9 +51,8 @@ type TTFTPercentileExtractorConfig struct { WindowSize int `json:"windowSize,omitempty"` // MaxObservationAge caps how far back the short window looks. // Observations older than this are never used for P50 or short-window P10. - MaxObservationAge string `json:"maxObservationAge,omitempty"` - InflightEMAAlpha float64 `json:"inflightEmaAlpha,omitempty"` - LowLoadWindowAge string `json:"lowLoadWindowAge,omitempty"` + MaxObservationAge string `json:"maxObservationAge,omitempty"` + LowLoadWindowAge string `json:"lowLoadWindowAge,omitempty"` // MaxRequests caps the short window to the most recent N observations regardless of age. MaxRequests int `json:"maxRequests,omitempty"` // MinRequests is the minimum capped-window count for the scorer to use the trusted @@ -64,19 +62,44 @@ type TTFTPercentileExtractorConfig struct { // TTFTPercentileMetrics is written to each model's attribute store every intervalDuration. type TTFTPercentileMetrics struct { - Requests int64 - AvgInflight float64 - InflightAtP50 float64 // avg inflight_at_dispatch of observations in the P40-P60 band - P10LowTTFT float64 // two-level P10: P10 of the bottom decile; hardware-floor estimate - P10TTFT float64 // P10 from capped short window - P50TTFT float64 // P50 from capped short window + Requests int64 + InflightAtP50 float64 // avg inflight_at_dispatch of observations in the P40-P60 band + P10LowTTFT float64 // two-level P10: P10 of the bottom decile; hardware-floor estimate + P10TTFT float64 // P10 from capped short window + P50TTFT float64 // P50 from capped short window LastObservedAt int64 - RecentN int // count of observations in the capped short window - MinRequests int // scorer threshold — copied from config so the scorer needs no separate param + RecentN int // count of observations in the capped short window + MinRequests int // scorer threshold — copied from config so the scorer needs no separate param } func (m TTFTPercentileMetrics) Clone() datalayer.Cloneable { return m } +// Floor is the load-invariant service floor: P10Low, or P10 before the long window fills. +// Zero means the model is truly cold. +func (m TTFTPercentileMetrics) Floor() float64 { + if m.P10LowTTFT > 0 { + return m.P10LowTTFT + } + return m.P10TTFT +} + +// Predict returns the effective TTFT at the current inflight and whether it is trusted +// (calibrated). Uncalibrated but observed → floor as an optimistic seed; cold → (0, false). +func (m TTFTPercentileMetrics) Predict() (effectiveTTFT float64, trusted bool) { + floor := m.Floor() + if floor == 0 { + return 0, false + } + if m.RecentN >= m.MinRequests && m.InflightAtP50 > 0 && m.P50TTFT > floor { + eff := floor + float64(m.Requests)*(m.P50TTFT-floor)/m.InflightAtP50 + if eff < floor { + eff = floor + } + return eff, true + } + return floor, false +} + type pendingEntry struct { inflightAtDispatch int64 dispatchedAt time.Time @@ -84,10 +107,9 @@ type pendingEntry struct { type modelPercentileState struct { TTFTPercentileMetrics - intervalStart time.Time - tracker percentileTracker - pending map[string]pendingEntry - avgInflightInit bool + intervalStart time.Time + tracker percentileTracker + pending map[string]pendingEntry } func (s *modelPercentileState) flush(now time.Time, maxObservationAge, lowLoadWindowAge time.Duration, maxRequests int) { @@ -113,7 +135,6 @@ type TTFTPercentileExtractor struct { windowSize int maxObservationAge time.Duration intervalDuration time.Duration - inflightEMAAlpha float64 lowLoadWindowAge time.Duration maxRequests int minRequests int @@ -124,7 +145,6 @@ func ExtractorFactory(name string, parameters json.RawMessage, h plugin.Handle) IntervalDuration: defaultIntervalDuration.String(), WindowSize: defaultWindowSize, MaxObservationAge: defaultMaxObservationAge.String(), - InflightEMAAlpha: defaultInflightEMAAlpha, LowLoadWindowAge: defaultLowLoadWindowAge.String(), MaxRequests: defaultMaxRequests, MinRequests: defaultMinRequests, @@ -143,9 +163,6 @@ func ExtractorFactory(name string, parameters json.RawMessage, h plugin.Handle) if cfg.MinRequests <= 0 { return nil, fmt.Errorf("minRequests must be > 0 for plugin %q", name) } - if cfg.InflightEMAAlpha <= 0 || cfg.InflightEMAAlpha > 1 { - return nil, fmt.Errorf("inflightEmaAlpha must be in (0,1] for plugin %q", name) - } interval, err := time.ParseDuration(cfg.IntervalDuration) if err != nil { return nil, fmt.Errorf("invalid intervalDuration %q for plugin %q: %w", cfg.IntervalDuration, name, err) @@ -162,7 +179,6 @@ func ExtractorFactory(name string, parameters json.RawMessage, h plugin.Handle) WithName(name). WithIntervalDuration(interval). WithWindow(cfg.WindowSize, maxObsAge). - WithInflightEMAAlpha(cfg.InflightEMAAlpha). WithLowLoadWindowAge(lowLoadAge). WithRequestBounds(cfg.MaxRequests, cfg.MinRequests), nil } @@ -175,7 +191,6 @@ func NewTTFTPercentileExtractor(ds datalayer.Datastore) *TTFTPercentileExtractor windowSize: defaultWindowSize, maxObservationAge: defaultMaxObservationAge, intervalDuration: defaultIntervalDuration, - inflightEMAAlpha: defaultInflightEMAAlpha, lowLoadWindowAge: defaultLowLoadWindowAge, maxRequests: defaultMaxRequests, minRequests: defaultMinRequests, @@ -192,9 +207,6 @@ func (e *TTFTPercentileExtractor) WithIntervalDuration(d time.Duration) *TTFTPer func (e *TTFTPercentileExtractor) WithWindow(size int, maxObsAge time.Duration) *TTFTPercentileExtractor { e.windowSize, e.maxObservationAge = size, maxObsAge; return e } -func (e *TTFTPercentileExtractor) WithInflightEMAAlpha(a float64) *TTFTPercentileExtractor { - e.inflightEMAAlpha = a; return e -} func (e *TTFTPercentileExtractor) WithLowLoadWindowAge(age time.Duration) *TTFTPercentileExtractor { e.lowLoadWindowAge = age; return e } @@ -258,12 +270,6 @@ func (e *TTFTPercentileExtractor) Extract(ctx context.Context, events []dlsrc.Ev delete(s.pending, reqID) } if now.Sub(s.intervalStart) >= e.intervalDuration { - sample := float64(s.Requests) - if !s.avgInflightInit { - s.AvgInflight, s.avgInflightInit = sample, true - } else { - s.AvgInflight = e.inflightEMAAlpha*sample + (1-e.inflightEMAAlpha)*s.AvgInflight - } s.flush(now, e.maxObservationAge, e.lowLoadWindowAge, e.maxRequests) } updated[model] = true @@ -282,22 +288,9 @@ func (e *TTFTPercentileExtractor) Extract(ctx context.Context, events []dlsrc.Ev m.MinRequests = e.minRequests e.ds.GetOrCreateModel(model).GetAttributes().Put(AttributeKey, m) - // Compute effectiveTTFT for the debug log (mirrors scorer logic). - floor := m.P10LowTTFT - if floor == 0 { - floor = m.P10TTFT - } - var eff float64 - if floor > 0 && m.RecentN >= m.MinRequests && m.InflightAtP50 > 0 && m.P50TTFT > floor { - eff = floor + float64(m.Requests)*(m.P50TTFT-floor)/m.InflightAtP50 - if eff < floor { - eff = floor - } - } else { - eff = floor // optimistic seed or unobserved - } + eff, _ := m.Predict() debugLogger.Info("ttft-percentile wrote attribute", - "model", model, "Requests", m.Requests, "AvgInflight", m.AvgInflight, + "model", model, "Requests", m.Requests, "InflightAtP50", m.InflightAtP50, "P10Low_s", m.P10LowTTFT, "P10_s", m.P10TTFT, "P50_s", m.P50TTFT, "EffectiveTTFT_s", eff, "RecentN", m.RecentN, "MinRequests", m.MinRequests, diff --git a/pkg/framework/plugins/modelselector/scorer/queuettft/README.md b/pkg/framework/plugins/modelselector/scorer/queuettft/README.md index 6d94345c..5191ec8d 100644 --- a/pkg/framework/plugins/modelselector/scorer/queuettft/README.md +++ b/pkg/framework/plugins/modelselector/scorer/queuettft/README.md @@ -6,18 +6,17 @@ Routes each request to the model with the lowest predicted TTFT under current lo Every TTFT decomposes as `TTFT = prefill_time + queue_wait`. -**P10Low** — load-invariant service floor: +**P10Low** — hardware-bound service floor: Computed from a long window (default 1h) using all observations regardless of inflight level: 1. Find the P10 TTFT threshold across all observations in the window -2. Take the P10 of only the observations at or below that threshold +2. Take the P10 of only the observations at or below that threshold (~P1 of all) -This isolates the fastest requests in the window — those with the least queue wait — without requiring the model to have idle periods. P10Low is invariant to load: it does not change with queue depth, concurrency, or scale events, because prefill time itself doesn't. +This isolates the fastest requests in the window — those with the least queue wait — without +requiring the model to have idle periods. P10Low is hardware-bound and stable: prefill time +does not change with queue depth, concurrency level, or scale events. -Short-prompt bias usually cancels: the scorer only ranks, so a shared bias is harmless. - - -**P50 and inflightAtP50** — current operating point (short window, default 3m): +**P50 and inflightAtP50** — current operating point (short window, default 3m / 100 requests): ``` P50 = 50th percentile TTFT inflightAtP50 = average inflight_at_dispatch of observations in the P40-P60 band @@ -35,7 +34,10 @@ Falls back to P10Low when P50 is not yet available or equals the floor. ``` score = (maxTTFT - effectiveTTFT) / (maxTTFT - minTTFT) ``` -Unobserved models score 1.0 (cold start) or 0.5 (idle alongside observed peers). +Under-observed models (UNOBSERVED or SEED state) receive an optimistic high score. +With `explorationRate > 0`, that high score is suppressed to 0 with probability +`(1 - explorationRate)` so only ~`explorationRate` fraction of requests are routed +to the under-observed model for calibration probing. ## Why it works physically @@ -48,16 +50,26 @@ The scorer draws a straight line through two points it has actually observed: - when there is no queue (`inflight = 0`): TTFT = P10Low (just the raw prefill time) - at the recent median load (`inflight = inflightAtP50`): TTFT = P50 -It then reads off that line at the current inflight to predict the next request's TTFT. +It then reads off that line at the current inflight to predict what the next request +will wait. No fitting, no tunable parameters — just two observed points. ## Parameters +### Scorer (`queue-ttft-scorer`) + +| Parameter | Default | Description | +|---|---|---| +| `explorationRate` | 0.0 | Fraction of requests routed to under-observed models for calibration probing. 0 = all requests go to the trusted winner; 0.1 = ~10% probe the under-observed model. | + +### Extractor (`ttft-percentile-extractor`) + | Parameter | Default | Description | |---|---|---| -| `windowAge` | 3m | Window for P50 (short -- keeps P50 fresh and responsive) | -| `lowLoadWindowAge` | 1h | Window for two-level P10Low | +| `maxObservationAge` | 3m | Time bound for the short window (P50 / P10) | +| `maxRequests` | 100 | Cap the short window to the most recent N observations | +| `minRequests` | 10 | Minimum capped-window count before the scorer trusts the formula | +| `lowLoadWindowAge` | 1h | Window for two-level P10Low (long = stable hardware floor) | | `windowSize` | 5000 | Ring buffer capacity (~200 KB per model) | -| `minObservations` | 3 | Minimum observations required to compute any percentile | ## Possible Enhancements @@ -72,15 +84,16 @@ P(model i) proportional to score_i^(1/T) # T = temperature, default 1.0 ``` At T = 1.0, a model scoring 0.8 vs 0.2 receives ~80% vs 20% of requests. - ### Prompt-length-aware floor P10Low is estimated from the fastest observed completions, which tend to be short-prompt -requests. For a long-prompt request, the prefill time is intrinsically higher, so the scorer under-predicts TTFT even at zero queue depth. +requests. For a long-prompt request, the hardware-floor prefill time is intrinsically higher, +so the scorer under-predicts TTFT even at zero queue depth. A more accurate floor would scale with the incoming prompt token count: ``` P10Low(tokens) = base_prefill + tokens × prefill_rate ``` where `base_prefill` and `prefill_rate` are fit from observations bucketed by prompt length. -This matters most when the workload has high prompt-length variance. +This matters most when the workload has high prompt-length variance (e.g. RAG pipelines +mixing short queries with large context windows). diff --git a/pkg/framework/plugins/modelselector/scorer/queuettft/plugin.go b/pkg/framework/plugins/modelselector/scorer/queuettft/plugin.go index d6c083db..7d7883c6 100644 --- a/pkg/framework/plugins/modelselector/scorer/queuettft/plugin.go +++ b/pkg/framework/plugins/modelselector/scorer/queuettft/plugin.go @@ -40,8 +40,6 @@ import ( const ( PluginType = "queue-ttft-scorer" - // unobserved is a sentinel returned by effectiveTTFT when no floor exists yet. - unobserved = -1.0 defaultExplorationRate = 0.0 // off by default; set e.g. 0.1 for 10% exploration @@ -105,204 +103,121 @@ func (s *QueueTTFTScorer) WithExplorationRate(r float64) *QueueTTFTScorer { s.explorationRate = r; return s } -// Score returns (maxTTFT − effectiveTTFT) / (maxTTFT − minTTFT) per model. -// -// Three cases per model: -// 1. Truly cold (no floor): unobserved sentinel → seeded to min(observed) after all are computed. -// 2. Trusted: RecentN ≥ MinRequests with a calibrated P50 → formula score. -// 3. Gone-quiet / under-observed: floor only (optimistic, assumes no queue). -// -// If all models are cold → all score 1.0. +// modelEval is the scorer's per-model working state for one Score call. +type modelEval struct { + metrics ttftpercentile.TTFTPercentileMetrics + eff float64 + trusted bool // calibrated operating point + observed bool // has a service floor (not truly cold) + suppressed bool // exploration-suppressed this cycle +} + +// Score ranks models by predicted TTFT: score = (maxTTFT − effectiveTTFT) / (maxTTFT − minTTFT). // -// If explorationRate > 0, models in state 1 or 3 are suppressed to score 0 with probability -// (1 - explorationRate). This throttles probing traffic to ~explorationRate of requests, -// preventing a burst of traffic to under-observed models before their first responses return. +// Cold models (no floor) are seeded at the best observed TTFT; if every model is cold, all +// score 1.0. With explorationRate > 0, under-observed models (not yet calibrated) are +// suppressed to 0 with probability (1 - explorationRate) to throttle probing traffic. func (s *QueueTTFTScorer) Score(ctx context.Context, cycleState *plugin.CycleState, _ *requesthandling.InferenceRequest, models []datalayer.Model) map[datalayer.Model]float64 { - effs := make(map[datalayer.Model]float64, len(models)) - needsProbe := make(map[datalayer.Model]bool, len(models)) - wasUnobserved := make(map[datalayer.Model]bool, len(models)) - minEff, maxEff := math.MaxFloat64, 0.0 - allUnobserved := true + evals := make(map[datalayer.Model]*modelEval, len(models)) + minEff := math.MaxFloat64 + anyObserved := false for _, model := range models { - v, probe := s.effectiveTTFT(ctx, model) - effs[model] = v - needsProbe[model] = probe - if v == unobserved { - wasUnobserved[model] = true - } else { - allUnobserved = false - if v > maxEff { - maxEff = v - } - if v < minEff { - minEff = v + m := metricsFor(model) + eff, trusted := m.Predict() + e := &modelEval{metrics: m, eff: eff, trusted: trusted, observed: m.Floor() > 0} + evals[model] = e + if e.observed { + anyObserved = true + if eff < minEff { + minEff = eff } } } - // All models cold → explore equally (no suppression). - if allUnobserved { - scores := make(map[datalayer.Model]float64, len(models)) + scores := make(map[datalayer.Model]float64, len(models)) + + // No model has a floor yet → nothing to rank; explore all equally. + if !anyObserved { for _, model := range models { scores[model] = 1.0 } - s.writeDecisions(cycleState, models, effs, wasUnobserved, needsProbe, nil) + s.writeDecisions(cycleState, models, evals) return scores } - // Seed truly cold models at the best observed TTFT (optimistic: assume as-good-as-best). - seed := minEff - for _, model := range models { - if effs[model] == unobserved { - effs[model] = seed - log.FromContext(ctx).V(logutil.DEBUG).Info("queue-ttft cold-seed", - "model", model.GetName(), "seed_s", seed, - ) - } - } - - // Recompute range after seeding (seed == minEff so max is unchanged, but be explicit). - minEff, maxEff = math.MaxFloat64, 0.0 - for _, v := range effs { - if v > maxEff { - maxEff = v + // Seed cold models at the best observed TTFT (optimistic). Seeds equal minEff, so the + // range spans the observed models and maxEff is their slowest. + maxEff := 0.0 + for _, e := range evals { + if !e.observed { + e.eff = minEff } - if v < minEff { - minEff = v + if e.eff > maxEff { + maxEff = e.eff } } - scores := make(map[datalayer.Model]float64, len(models)) for _, model := range models { + e := evals[model] if maxEff == minEff { scores[model] = 1.0 } else { - scores[model] = (maxEff - effs[model]) / (maxEff - minEff) + scores[model] = (maxEff - e.eff) / (maxEff - minEff) } - } - - explorationSuppressed := make(map[datalayer.Model]bool, len(models)) - if s.explorationRate > 0 { - for _, model := range models { - if needsProbe[model] && rand.Float64() >= s.explorationRate { - scores[model] = 0 - explorationSuppressed[model] = true - log.FromContext(ctx).V(logutil.DEBUG).Info("queue-ttft exploration suppressed", - "model", model.GetName(), "explorationRate", s.explorationRate, - ) - } + if s.explorationRate > 0 && !e.trusted && rand.Float64() >= s.explorationRate { + scores[model] = 0 + e.suppressed = true } } if dl := log.FromContext(ctx).V(logutil.DEBUG); dl.Enabled() { for _, model := range models { - dl.Info("queue-ttft score", - "model", model.GetName(), - "effectiveTTFT", effs[model], - "score", scores[model], - "needsProbe", needsProbe[model], - ) + e := evals[model] + dl.Info("queue-ttft score", "model", model.GetName(), + "effectiveTTFT", e.eff, "score", scores[model], "trusted", e.trusted) } } - s.writeDecisions(cycleState, models, effs, wasUnobserved, needsProbe, explorationSuppressed) + s.writeDecisions(cycleState, models, evals) return scores } +// metricsFor reads the TTFT percentile metrics an extractor published for the model. +// A missing or malformed attribute yields the zero value, which reads as truly cold. +func metricsFor(model datalayer.Model) ttftpercentile.TTFTPercentileMetrics { + if val, ok := model.GetAttributes().Get(ttftpercentile.AttributeKey); ok { + if m, ok := val.(ttftpercentile.TTFTPercentileMetrics); ok { + return m + } + } + return ttftpercentile.TTFTPercentileMetrics{} +} + // writeDecisions stores per-model scoring decisions in CycleState for response plugins. -func (s *QueueTTFTScorer) writeDecisions(cycleState *plugin.CycleState, models []datalayer.Model, - effs map[datalayer.Model]float64, wasUnobserved, needsProbe, explorationSuppressed map[datalayer.Model]bool) { +func (s *QueueTTFTScorer) writeDecisions(cycleState *plugin.CycleState, models []datalayer.Model, evals map[datalayer.Model]*modelEval) { if cycleState == nil { return } decisions := make(map[string]ScoringDecision, len(models)) for _, model := range models { - var state string + e := evals[model] + state := "trusted" switch { - case wasUnobserved[model]: + case !e.observed: state = "unobserved" - case needsProbe[model]: + case !e.trusted: state = "seed" - default: - state = "trusted" } - - var floor, p50 float64 - var recentN int - var inflight int64 - if val, ok := model.GetAttributes().Get(ttftpercentile.AttributeKey); ok { - if m, ok := val.(ttftpercentile.TTFTPercentileMetrics); ok { - floor = m.P10LowTTFT - if floor == 0 { - floor = m.P10TTFT - } - p50 = m.P50TTFT - recentN = m.RecentN - inflight = m.Requests - } - } - decisions[model.GetName()] = ScoringDecision{ - EffectiveTTFT: effs[model], - Floor: floor, - P50: p50, - RecentN: recentN, - Inflight: inflight, + EffectiveTTFT: e.eff, + Floor: e.metrics.Floor(), + P50: e.metrics.P50TTFT, + RecentN: e.metrics.RecentN, + Inflight: e.metrics.Requests, State: state, - ExplorationSuppressed: explorationSuppressed[model], + ExplorationSuppressed: e.suppressed, } } cycleState.Write(DecisionsCycleStateKey, decisions) } - -// effectiveTTFT returns the predicted TTFT for the model's next request and whether -// the model needs exploration (UNOBSERVED or SEED state — not yet calibrated). -// -// - (unobserved, true): no floor yet — truly cold model. -// - (floor, true): under-observed or gone-quiet — optimistic seed, assume no queue. -// - (floor + inflight×slope, false): trusted operating point with recent calibration. -func (s *QueueTTFTScorer) effectiveTTFT(ctx context.Context, model datalayer.Model) (float64, bool) { - val, ok := model.GetAttributes().Get(ttftpercentile.AttributeKey) - if !ok { - return unobserved, true - } - m, ok := val.(ttftpercentile.TTFTPercentileMetrics) - if !ok { - return unobserved, true - } - - floor := m.P10LowTTFT - if floor == 0 { - floor = m.P10TTFT - } - if floor == 0 { - return unobserved, true // truly cold: no hardware floor established yet - } - - // Trusted: enough recent observations and a calibrated operating point. - if m.RecentN >= m.MinRequests && m.InflightAtP50 > 0 && m.P50TTFT > floor { - eff := floor + float64(m.Requests)*(m.P50TTFT-floor)/m.InflightAtP50 - if eff < floor { - eff = floor - } - if dl := log.FromContext(ctx).V(logutil.DEBUG); dl.Enabled() { - dl.Info("queue-ttft effective trusted", - "model", model.GetName(), "inflight", m.Requests, - "inflightAtP50", m.InflightAtP50, "P10Low_s", m.P10LowTTFT, - "P50_s", m.P50TTFT, "recentN", m.RecentN, "effectiveTTFT", eff, - ) - } - return eff, false - } - - // Optimistic seed: model is under-observed or gone-quiet. - // Assume no queue — predict only the hardware floor. - if dl := log.FromContext(ctx).V(logutil.DEBUG); dl.Enabled() { - dl.Info("queue-ttft effective seed", - "model", model.GetName(), "floor_s", floor, - "recentN", m.RecentN, "minRequests", m.MinRequests, - ) - } - return floor, true -} From 8cd6e70cdff4f2bd69f1a418453dee100678b9f5 Mon Sep 17 00:00:00 2001 From: Mohammad Date: Wed, 1 Jul 2026 19:28:26 +0300 Subject: [PATCH 11/27] Add percentile function. Signed-off-by: Mohammad --- .../datalayer/ttftpercentile/tracker.go | 59 +++++++++---------- 1 file changed, 27 insertions(+), 32 deletions(-) diff --git a/pkg/framework/plugins/datalayer/ttftpercentile/tracker.go b/pkg/framework/plugins/datalayer/ttftpercentile/tracker.go index 85b8f93f..c42b4bc0 100644 --- a/pkg/framework/plugins/datalayer/ttftpercentile/tracker.go +++ b/pkg/framework/plugins/datalayer/ttftpercentile/tracker.go @@ -90,22 +90,32 @@ func (t *slidingWindowTracker) countCapped(now time.Time, maxAge time.Duration, return len(t.recentObsCapped(now, maxAge, maxN)) } -func (t *slidingWindowTracker) quantile(p float64, now time.Time, maxAge time.Duration, maxN int) (float64, bool) { - obs := t.recentObsCapped(now, maxAge, maxN) - if len(obs) == 0 { - return 0, false +// percentileOf returns the p-th percentile of sorted (ascending) by linear interpolation. +func percentileOf(sorted []float64, p float64) float64 { + idx := p * float64(len(sorted)-1) + lo := int(idx) + if lo+1 >= len(sorted) { + return sorted[lo] } + return sorted[lo] + (idx-float64(lo))*(sorted[lo+1]-sorted[lo]) +} + +// sortedValues extracts and sorts (ascending) the observation values. +func sortedValues(obs []observation) []float64 { vals := make([]float64, len(obs)) for i, o := range obs { vals[i] = o.value } sort.Float64s(vals) - idx := p * float64(len(vals)-1) - lo := int(idx) - if lo+1 >= len(vals) { - return vals[lo], true + return vals +} + +func (t *slidingWindowTracker) quantile(p float64, now time.Time, maxAge time.Duration, maxN int) (float64, bool) { + obs := t.recentObsCapped(now, maxAge, maxN) + if len(obs) == 0 { + return 0, false } - return vals[lo] + (idx-float64(lo))*(vals[lo+1]-vals[lo]), true + return percentileOf(sortedValues(obs), p), true } func (t *slidingWindowTracker) p10Low(now time.Time, maxAge time.Duration) (float64, bool) { @@ -114,25 +124,14 @@ func (t *slidingWindowTracker) p10Low(now time.Time, maxAge time.Duration) (floa if len(obs) == 0 { return 0, false } - vals := make([]float64, len(obs)) - for i, o := range obs { - vals[i] = o.value - } - sort.Float64s(vals) + vals := sortedValues(obs) // Level 1: P10 threshold — index of the 10th-percentile observation. lo := int(0.10 * float64(len(vals)-1)) - // Level 2: P10 of the bottom-decile slice (vals[:lo+1], already sorted). - // This is ~P1 of all observations: the fastest requests in the window - // regardless of their inflight count, approximating the hardware floor. - low := vals[:lo+1] - idx2 := 0.10 * float64(len(low)-1) - lo2 := int(idx2) - if lo2+1 >= len(low) { - return low[lo2], true - } - return low[lo2] + (idx2-float64(lo2))*(low[lo2+1]-low[lo2]), true + // Level 2: P10 of the bottom-decile slice (~P1 of all observations): the fastest + // requests in the window regardless of inflight, approximating the hardware floor. + return percentileOf(vals[:lo+1], 0.10), true } func (t *slidingWindowTracker) quantileWithInflight(p float64, now time.Time, maxAge time.Duration, maxN int) (float64, float64, bool) { @@ -142,15 +141,11 @@ func (t *slidingWindowTracker) quantileWithInflight(p float64, now time.Time, ma } sort.Slice(obs, func(i, j int) bool { return obs[i].value < obs[j].value }) n := len(obs) - - idx := p * float64(n-1) - lo := int(idx) - var value float64 - if lo+1 >= n { - value = obs[lo].value - } else { - value = obs[lo].value + (idx-float64(lo))*(obs[lo+1].value-obs[lo].value) + vals := make([]float64, n) + for i, o := range obs { + vals[i] = o.value } + value := percentileOf(vals, p) // Average inflight of observations in the [p-0.1, p+0.1] band. // Using a band rather than a single point makes inflightAtP50 more stable. From c4001fc12477c10a0d46bf15a3d8f8864be3dabd Mon Sep 17 00:00:00 2001 From: Mohammad Date: Wed, 1 Jul 2026 19:32:19 +0300 Subject: [PATCH 12/27] Remove python script. Signed-off-by: Mohammad --- examples/plot_capacity.py | 162 -------------------------------------- 1 file changed, 162 deletions(-) delete mode 100644 examples/plot_capacity.py diff --git a/examples/plot_capacity.py b/examples/plot_capacity.py deleted file mode 100644 index 7f416aec..00000000 --- a/examples/plot_capacity.py +++ /dev/null @@ -1,162 +0,0 @@ -""" -Plot effective TTFT vs real TTFT from a ttft-percentile extractor log file. - -Produces one file with 4 panels: - 1. effectiveTTFT line + real TTFT scatter - 2. effectiveTTFT line + real TTFT binned median line - 3. Inflight requests - 4. P10Low and P50 TTFT lines - -Usage: python plot_capacity.py -""" - -import json -import statistics -import sys -from collections import defaultdict - -import matplotlib.pyplot as plt - -LOG_FILE = sys.argv[1] if len(sys.argv) > 1 else "logs/bench_median33" -MAX_T_SEC = float(sys.argv[2]) if len(sys.argv) > 2 else None # optional time cutoff -BIN_SEC = 10 # bucket width for the median graph - -# --------------------------------------------------------------------------- -# Parse -# --------------------------------------------------------------------------- -flush_records = defaultdict(list) -obs_records = defaultdict(list) - -with open(LOG_FILE) as f: - for line in f: - line = line.strip() - if not line: - continue - try: - obj = json.loads(line) - except json.JSONDecodeError: - continue - msg = obj.get("msg", "") - model = obj.get("model", "") - if not model: - continue - if msg == "ttft-percentile wrote attribute": - flush_records[model].append({ - "ts": obj["ts"], - "inflight": obj.get("Requests", 0), - "inflight_at_p50": obj.get("InflightAtP50", 0), - "p10low": obj.get("P10Low_s", 0), - "p50": obj.get("P50_s", 0), - "effective": obj.get("EffectiveTTFT_s", 0), - }) - elif msg == "ttft-observation": - ttft = obj.get("ttft_s", 0) - obs_records[model].append({ - "ts": obj["ts"] - ttft, # align to dispatch time (response_ts - ttft) - "ttft": ttft, - }) - -if not flush_records: - print("No 'ttft-percentile wrote attribute' lines found in", LOG_FILE) - sys.exit(1) - -# Normalise timestamps -t0 = min( - r["ts"] - for recs in list(flush_records.values()) + list(obs_records.values()) - for r in recs -) -for recs in flush_records.values(): - for r in recs: - r["t"] = r["ts"] - t0 -for recs in obs_records.values(): - for r in recs: - r["t"] = r["ts"] - t0 - -if MAX_T_SEC is not None: - flush_records = {m: [r for r in recs if r["t"] <= MAX_T_SEC] for m, recs in flush_records.items()} - obs_records = {m: [r for r in recs if r["t"] <= MAX_T_SEC] for m, recs in obs_records.items()} - -models = sorted(flush_records.keys()) -colors = ["steelblue", "tomato", "seagreen", "darkorange"] -base = LOG_FILE.rstrip("/").split("/")[-1] - -fig, axes = plt.subplots(6, 1, figsize=(14, 20), sharex=True) -fig.suptitle(f"Effective TTFT vs Real TTFT — {base}", fontsize=13) -ax_scatter, ax_median, ax_inflight, ax_ttft_stat, ax_load_ratio, ax_spread = axes - -for i, model in enumerate(models): - color = colors[i % len(colors)] - label = model.split("/")[-1] - frecs = flush_records[model] - orecs = obs_records.get(model, []) - ft = [r["t"] for r in frecs] - fe = [r["effective"] for r in frecs] - fp = [r["p10low"] for r in frecs] - fp50 = [r["p50"] for r in frecs] - fi = [r["inflight"] for r in frecs] - fip50 = [r["inflight_at_p50"] for r in frecs] - # load ratio: inflight / inflightAtP50 (capped to avoid div-by-zero noise) - fratio = [inf / ip50 if ip50 > 0 else 0 for inf, ip50 in zip(fi, fip50)] - # spread: P50 - P10Low (the TTFT range the formula scales over) - fspread = [p50 - p10 for p50, p10 in zip(fp50, fp)] - - # Panel 1: scatter - ax_scatter.plot(ft, fe, color=color, linewidth=1.2, alpha=0.9, - linestyle="-", label=f"{label} effectiveTTFT") - if orecs: - ot = [r["t"] for r in orecs] - ov = [r["ttft"] for r in orecs] - ax_scatter.scatter(ot, ov, color=color, alpha=0.15, s=3, - label=f"{label} real TTFT") - - # Panel 2: binned median line - ax_median.plot(ft, fe, color=color, linewidth=1.2, alpha=0.9, - linestyle="-", label=f"{label} effectiveTTFT") - if orecs: - bins = defaultdict(list) - for r in orecs: - bins[int(r["t"] // BIN_SEC)].append(r["ttft"]) - bt = sorted(bins) - bv = [statistics.median(bins[b]) for b in bt] - ax_median.plot([b * BIN_SEC + BIN_SEC / 2 for b in bt], bv, - color=color, linewidth=1.2, alpha=0.7, - linestyle="--", label=f"{label} real TTFT (median/{BIN_SEC}s)") - - # Panel 3: inflight - ax_inflight.plot(ft, fi, color=color, alpha=0.7, linewidth=0.8, label=label) - - # Panel 4: P10Low and P50 - ax_ttft_stat.plot(ft, fp, color=color, alpha=0.9, linewidth=1.0, - linestyle="-", label=f"{label} P10Low") - ax_ttft_stat.plot(ft, fp50, color=color, alpha=0.5, linewidth=0.8, - linestyle="--", label=f"{label} P50") - - # Panel 5: load ratio = inflight / inflightAtP50 - ax_load_ratio.plot(ft, fratio, color=color, linewidth=1.0, alpha=0.85, label=label) - ax_load_ratio.axhline(1.0, color="gray", linewidth=0.7, linestyle=":") - - # Panel 6: spread = P50 - P10Low - ax_spread.plot(ft, fspread, color=color, linewidth=1.0, alpha=0.85, label=label) - -for ax, title, ylabel in [ - (ax_scatter, "effectiveTTFT vs real TTFT (scatter)", "TTFT (s)"), - (ax_median, f"effectiveTTFT vs real TTFT (median/{BIN_SEC}s bins)", "TTFT (s)"), - (ax_inflight, "Inflight requests", "Inflight"), - (ax_ttft_stat, "P10Low and P50", "TTFT (s)"), - (ax_load_ratio, "Load ratio: inflight / inflightAtP50 (1.0 = median load)", "ratio"), - (ax_spread, "Spread: P50 − P10Low (TTFT range the slope scales over)", "TTFT (s)"), -]: - ax.set_ylabel(ylabel) - ax.set_ylim(bottom=0) - ax.legend(fontsize=8, ncol=2) - ax.grid(True, alpha=0.3) - ax.set_title(title, fontsize=9, loc="left", pad=3) - -ax_ttft_stat.set_xlabel("Time since first request (s)") - -plt.tight_layout() -out = base + "_ttft_cmp.png" -fig.savefig(out, dpi=150) -print("saved to", out) -plt.show() From 6ff7eb1a369abd9d884754e5cc48dfe1c76f0ff6 Mon Sep 17 00:00:00 2001 From: Mohammad Date: Wed, 1 Jul 2026 23:52:09 +0300 Subject: [PATCH 13/27] Format the code. Signed-off-by: Mohammad --- .../datalayer/ttftpercentile/plugin.go | 66 ++++---- .../datalayer/ttftpercentile/tracker.go | 145 ++++++------------ .../modelselector/scorer/queuettft/plugin.go | 6 +- 3 files changed, 85 insertions(+), 132 deletions(-) diff --git a/pkg/framework/plugins/datalayer/ttftpercentile/plugin.go b/pkg/framework/plugins/datalayer/ttftpercentile/plugin.go index 64281807..1338b605 100644 --- a/pkg/framework/plugins/datalayer/ttftpercentile/plugin.go +++ b/pkg/framework/plugins/datalayer/ttftpercentile/plugin.go @@ -47,8 +47,8 @@ const ( var _ dlsrc.Extractor = &TTFTPercentileExtractor{} type TTFTPercentileExtractorConfig struct { - IntervalDuration string `json:"intervalDuration,omitempty"` - WindowSize int `json:"windowSize,omitempty"` + IntervalDuration string `json:"intervalDuration,omitempty"` + WindowSize int `json:"windowSize,omitempty"` // MaxObservationAge caps how far back the short window looks. // Observations older than this are never used for P50 or short-window P10. MaxObservationAge string `json:"maxObservationAge,omitempty"` @@ -108,23 +108,24 @@ type pendingEntry struct { type modelPercentileState struct { TTFTPercentileMetrics intervalStart time.Time - tracker percentileTracker + tracker *slidingWindowTracker pending map[string]pendingEntry } func (s *modelPercentileState) flush(now time.Time, maxObservationAge, lowLoadWindowAge time.Duration, maxRequests int) { - p50, inflightAtP50, ok50 := s.tracker.quantileWithInflight(0.50, now, maxObservationAge, maxRequests) - p10, ok10 := s.tracker.quantile(0.10, now, maxObservationAge, maxRequests) - if ok50 { - s.P50TTFT, s.InflightAtP50, s.LastObservedAt = p50, inflightAtP50, now.UnixNano() + // Short window (value-sorted, capped): one snapshot feeds P10, P50 and inflightAtP50. + short := s.tracker.window(now, maxObservationAge, maxRequests) + s.RecentN = len(short) + if len(short) > 0 { + s.P10TTFT = percentileOf(short, 0.10) + s.P50TTFT = percentileOf(short, 0.50) + s.InflightAtP50 = bandInflight(short, 0.50) + s.LastObservedAt = now.UnixNano() } - if ok10 { - s.P10TTFT = p10 + // Long window (uncapped): stable hardware floor. + if long := s.tracker.window(now, lowLoadWindowAge, 0); len(long) > 0 { + s.P10LowTTFT = twoLevelP10(long) } - if p10low, ok := s.tracker.p10Low(now, lowLoadWindowAge); ok { - s.P10LowTTFT = p10low - } - s.RecentN = s.tracker.countCapped(now, maxObservationAge, maxRequests) s.intervalStart = now } @@ -199,19 +200,24 @@ func NewTTFTPercentileExtractor(ds datalayer.Datastore) *TTFTPercentileExtractor func (e *TTFTPercentileExtractor) TypedName() plugin.TypedName { return e.typedName } func (e *TTFTPercentileExtractor) WithName(n string) *TTFTPercentileExtractor { - e.typedName.Name = n; return e + e.typedName.Name = n + return e } func (e *TTFTPercentileExtractor) WithIntervalDuration(d time.Duration) *TTFTPercentileExtractor { - e.intervalDuration = d; return e + e.intervalDuration = d + return e } func (e *TTFTPercentileExtractor) WithWindow(size int, maxObsAge time.Duration) *TTFTPercentileExtractor { - e.windowSize, e.maxObservationAge = size, maxObsAge; return e + e.windowSize, e.maxObservationAge = size, maxObsAge + return e } func (e *TTFTPercentileExtractor) WithLowLoadWindowAge(age time.Duration) *TTFTPercentileExtractor { - e.lowLoadWindowAge = age; return e + e.lowLoadWindowAge = age + return e } func (e *TTFTPercentileExtractor) WithRequestBounds(maxN, minN int) *TTFTPercentileExtractor { - e.maxRequests, e.minRequests = maxN, minN; return e + e.maxRequests, e.minRequests = maxN, minN + return e } func (e *TTFTPercentileExtractor) Extract(ctx context.Context, events []dlsrc.Event) error { @@ -262,9 +268,11 @@ func (e *TTFTPercentileExtractor) Extract(ctx context.Context, events []dlsrc.Ev inflightAtDispatch = entry.inflightAtDispatch } s.tracker.add(ttft, inflightAtDispatch, now) - debugLogger.Info("ttft-observation", - "model", model, "ttft_s", ttft, "inflightAtDispatch", inflightAtDispatch, - ) + if debugLogger.Enabled() { + debugLogger.Info("ttft-observation", + "model", model, "ttft_s", ttft, "inflightAtDispatch", inflightAtDispatch, + ) + } } if reqID != "" { delete(s.pending, reqID) @@ -288,13 +296,15 @@ func (e *TTFTPercentileExtractor) Extract(ctx context.Context, events []dlsrc.Ev m.MinRequests = e.minRequests e.ds.GetOrCreateModel(model).GetAttributes().Put(AttributeKey, m) - eff, _ := m.Predict() - debugLogger.Info("ttft-percentile wrote attribute", - "model", model, "Requests", m.Requests, - "InflightAtP50", m.InflightAtP50, "P10Low_s", m.P10LowTTFT, - "P10_s", m.P10TTFT, "P50_s", m.P50TTFT, "EffectiveTTFT_s", eff, - "RecentN", m.RecentN, "MinRequests", m.MinRequests, - ) + if debugLogger.Enabled() { + eff, _ := m.Predict() + debugLogger.Info("ttft-percentile wrote attribute", + "model", model, "Requests", m.Requests, + "InflightAtP50", m.InflightAtP50, "P10Low_s", m.P10LowTTFT, + "P10_s", m.P10TTFT, "P50_s", m.P50TTFT, "EffectiveTTFT_s", eff, + "RecentN", m.RecentN, "MinRequests", m.MinRequests, + ) + } } return nil } diff --git a/pkg/framework/plugins/datalayer/ttftpercentile/tracker.go b/pkg/framework/plugins/datalayer/ttftpercentile/tracker.go index c42b4bc0..fa7a31cb 100644 --- a/pkg/framework/plugins/datalayer/ttftpercentile/tracker.go +++ b/pkg/framework/plugins/datalayer/ttftpercentile/tracker.go @@ -21,37 +21,21 @@ import ( "time" ) -type percentileTracker interface { - add(value float64, inflight int64, ts time.Time) - // quantile returns the p-th percentile of the most recent min(maxN, available) - // observations within maxAge. Returns false if there are no observations. - quantile(p float64, now time.Time, maxAge time.Duration, maxN int) (float64, bool) - // quantileWithInflight returns the p-th percentile TTFT and the average inflight - // of observations in the [p-0.1, p+0.1] band of the capped short window. - // Returns false if there are no observations. - quantileWithInflight(p float64, now time.Time, maxAge time.Duration, maxN int) (float64, float64, bool) - // p10Low computes the two-level P10 over the full lowLoadWindowAge window (no cap). - // It isolates low-queue-wait observations without requiring a fixed inflight cut-off. - // Returns false if there are no observations. - p10Low(now time.Time, maxAge time.Duration) (float64, bool) - // countCapped returns the number of observations in the capped short window. - countCapped(now time.Time, maxAge time.Duration, maxN int) int -} - type observation struct { value float64 inflight int64 // inflight_at_dispatch ts time.Time } +// slidingWindowTracker is a fixed-capacity ring buffer of TTFT observations. type slidingWindowTracker struct { buf []observation head int n int } -func newSlidingWindowTracker(cap int) *slidingWindowTracker { - return &slidingWindowTracker{buf: make([]observation, cap)} +func newSlidingWindowTracker(capacity int) *slidingWindowTracker { + return &slidingWindowTracker{buf: make([]observation, capacity)} } func (t *slidingWindowTracker) add(value float64, inflight int64, ts time.Time) { @@ -62,106 +46,63 @@ func (t *slidingWindowTracker) add(value float64, inflight int64, ts time.Time) } } -// recentObs returns observations within maxAge, newest first. -func (t *slidingWindowTracker) recentObs(now time.Time, maxAge time.Duration) []observation { +// window returns the observations within maxAge, capped to the most recent maxN +// (0 = no cap), sorted ascending by value. Observations are stored in time order, +// so the newest-first scan stops at the first one older than the cutoff. +func (t *slidingWindowTracker) window(now time.Time, maxAge time.Duration, maxN int) []observation { cutoff := now.Add(-maxAge) - cap := len(t.buf) - out := make([]observation, 0, t.n) + size := len(t.buf) + capHint := t.n + if maxN > 0 && maxN < capHint { + capHint = maxN + } + out := make([]observation, 0, capHint) for i := 0; i < t.n; i++ { - obs := t.buf[(t.head-1-i+cap)%cap] - if !obs.ts.Before(cutoff) { - out = append(out, obs) + o := t.buf[(t.head-1-i+size)%size] + if o.ts.Before(cutoff) { + break + } + out = append(out, o) + if maxN > 0 && len(out) == maxN { + break } } + sort.Slice(out, func(i, j int) bool { return out[i].value < out[j].value }) return out } -// recentObsCapped returns the most recent min(maxN, available) observations within maxAge. -// recentObs is already newest-first so a prefix slice gives the most recent maxN. -func (t *slidingWindowTracker) recentObsCapped(now time.Time, maxAge time.Duration, maxN int) []observation { - obs := t.recentObs(now, maxAge) - if maxN > 0 && len(obs) > maxN { - obs = obs[:maxN] - } - return obs -} - -func (t *slidingWindowTracker) countCapped(now time.Time, maxAge time.Duration, maxN int) int { - return len(t.recentObsCapped(now, maxAge, maxN)) -} - -// percentileOf returns the p-th percentile of sorted (ascending) by linear interpolation. -func percentileOf(sorted []float64, p float64) float64 { +// percentileOf returns the p-th percentile value of a value-sorted slice by linear interpolation. +func percentileOf(sorted []observation, p float64) float64 { idx := p * float64(len(sorted)-1) lo := int(idx) if lo+1 >= len(sorted) { - return sorted[lo] - } - return sorted[lo] + (idx-float64(lo))*(sorted[lo+1]-sorted[lo]) -} - -// sortedValues extracts and sorts (ascending) the observation values. -func sortedValues(obs []observation) []float64 { - vals := make([]float64, len(obs)) - for i, o := range obs { - vals[i] = o.value + return sorted[lo].value } - sort.Float64s(vals) - return vals + return sorted[lo].value + (idx-float64(lo))*(sorted[lo+1].value-sorted[lo].value) } -func (t *slidingWindowTracker) quantile(p float64, now time.Time, maxAge time.Duration, maxN int) (float64, bool) { - obs := t.recentObsCapped(now, maxAge, maxN) - if len(obs) == 0 { - return 0, false - } - return percentileOf(sortedValues(obs), p), true +// twoLevelP10 estimates the hardware floor: the P10 of the bottom-decile slice (~P1 of all), +// isolating the fastest requests in the window regardless of their inflight count. +func twoLevelP10(sorted []observation) float64 { + lo := int(0.10 * float64(len(sorted)-1)) + return percentileOf(sorted[:lo+1], 0.10) } -func (t *slidingWindowTracker) p10Low(now time.Time, maxAge time.Duration) (float64, bool) { - // Uses the full lowLoadWindowAge window with no cap — stable hardware floor. - obs := t.recentObs(now, maxAge) - if len(obs) == 0 { - return 0, false +// bandInflight returns the average inflight of observations in the [p-0.1, p+0.1] band of +// a value-sorted slice. Averaging a band rather than a single point stabilises the estimate. +func bandInflight(sorted []observation, p float64) float64 { + n := len(sorted) + lo := int((p - 0.10) * float64(n-1)) + if lo < 0 { + lo = 0 } - vals := sortedValues(obs) - - // Level 1: P10 threshold — index of the 10th-percentile observation. - lo := int(0.10 * float64(len(vals)-1)) - - // Level 2: P10 of the bottom-decile slice (~P1 of all observations): the fastest - // requests in the window regardless of inflight, approximating the hardware floor. - return percentileOf(vals[:lo+1], 0.10), true -} - -func (t *slidingWindowTracker) quantileWithInflight(p float64, now time.Time, maxAge time.Duration, maxN int) (float64, float64, bool) { - obs := t.recentObsCapped(now, maxAge, maxN) - if len(obs) == 0 { - return 0, 0, false - } - sort.Slice(obs, func(i, j int) bool { return obs[i].value < obs[j].value }) - n := len(obs) - vals := make([]float64, n) - for i, o := range obs { - vals[i] = o.value - } - value := percentileOf(vals, p) - - // Average inflight of observations in the [p-0.1, p+0.1] band. - // Using a band rather than a single point makes inflightAtP50 more stable. - bandLo := int((p - 0.10) * float64(n-1)) - if bandLo < 0 { - bandLo = 0 - } - bandHi := int((p + 0.10) * float64(n-1)) - if bandHi >= n { - bandHi = n - 1 + hi := int((p + 0.10) * float64(n-1)) + if hi >= n { + hi = n - 1 } var sum float64 - for i := bandLo; i <= bandHi; i++ { - sum += float64(obs[i].inflight) + for i := lo; i <= hi; i++ { + sum += float64(sorted[i].inflight) } - avgInflight := sum / float64(bandHi-bandLo+1) - - return value, avgInflight, true + return sum / float64(hi-lo+1) } diff --git a/pkg/framework/plugins/modelselector/scorer/queuettft/plugin.go b/pkg/framework/plugins/modelselector/scorer/queuettft/plugin.go index 7d7883c6..41592afc 100644 --- a/pkg/framework/plugins/modelselector/scorer/queuettft/plugin.go +++ b/pkg/framework/plugins/modelselector/scorer/queuettft/plugin.go @@ -97,10 +97,12 @@ func NewQueueTTFTScorer() *QueueTTFTScorer { func (s *QueueTTFTScorer) TypedName() plugin.TypedName { return s.typedName } func (s *QueueTTFTScorer) WithName(name string) *QueueTTFTScorer { - s.typedName.Name = name; return s + s.typedName.Name = name + return s } func (s *QueueTTFTScorer) WithExplorationRate(r float64) *QueueTTFTScorer { - s.explorationRate = r; return s + s.explorationRate = r + return s } // modelEval is the scorer's per-model working state for one Score call. From 17b31c4f548b3d1c5989ceb9309070361492fc59 Mon Sep 17 00:00:00 2001 From: Mohammad Date: Thu, 2 Jul 2026 00:32:04 +0300 Subject: [PATCH 14/27] Fix comments, use cyclestate. Signed-off-by: Mohammad --- .../interface/datalayer/datasource/types.go | 12 ++++--- .../datalayer/ttftpercentile/plugin.go | 33 +++++-------------- pkg/handlers/request.go | 2 +- pkg/handlers/response.go | 9 ++--- 4 files changed, 22 insertions(+), 34 deletions(-) diff --git a/pkg/framework/interface/datalayer/datasource/types.go b/pkg/framework/interface/datalayer/datasource/types.go index 016605ce..b22f36f3 100644 --- a/pkg/framework/interface/datalayer/datasource/types.go +++ b/pkg/framework/interface/datalayer/datasource/types.go @@ -46,15 +46,17 @@ const ( // RequestPayload is the Payload for RequestEventType. type RequestPayload struct { - Request *requesthandling.InferenceRequest + Request *requesthandling.InferenceRequest + CycleState *plugin.CycleState } // ResponsePayload is the Payload for ResponseEventType. type ResponsePayload struct { - Request *requesthandling.InferenceRequest - Response *requesthandling.InferenceResponse - Duration time.Duration - TTFT time.Duration + Request *requesthandling.InferenceRequest + Response *requesthandling.InferenceResponse + CycleState *plugin.CycleState + Duration time.Duration + TTFT time.Duration } type DatalayerProcessor interface { diff --git a/pkg/framework/plugins/datalayer/ttftpercentile/plugin.go b/pkg/framework/plugins/datalayer/ttftpercentile/plugin.go index 1338b605..55a15c7e 100644 --- a/pkg/framework/plugins/datalayer/ttftpercentile/plugin.go +++ b/pkg/framework/plugins/datalayer/ttftpercentile/plugin.go @@ -36,6 +36,10 @@ const ( PluginType = "ttft-percentile-extractor" AttributeKey = "ttft-percentile" + // inflightAtDispatchKey carries a request's inflight-at-dispatch through CycleState + // from the request event to the matching response event. + inflightAtDispatchKey = "ttft-percentile/inflight-at-dispatch" + defaultWindowSize = 5000 defaultMaxObservationAge = 3 * time.Minute // observations older than this are never used defaultIntervalDuration = 5 * time.Second @@ -100,16 +104,10 @@ func (m TTFTPercentileMetrics) Predict() (effectiveTTFT float64, trusted bool) { return floor, false } -type pendingEntry struct { - inflightAtDispatch int64 - dispatchedAt time.Time -} - type modelPercentileState struct { TTFTPercentileMetrics intervalStart time.Time tracker *slidingWindowTracker - pending map[string]pendingEntry } func (s *modelPercentileState) flush(now time.Time, maxObservationAge, lowLoadWindowAge time.Duration, maxRequests int) { @@ -239,11 +237,9 @@ func (e *TTFTPercentileExtractor) Extract(ctx context.Context, events []dlsrc.Ev s := e.getOrCreate(model) inflight := s.Requests s.Requests++ - if reqID := p.Request.Headers["x-request-id"]; reqID != "" { - s.pending[reqID] = pendingEntry{ - inflightAtDispatch: inflight, - dispatchedAt: now, - } + // Stash inflight-at-dispatch in CycleState; the matching response event reads it back. + if p.CycleState != nil { + p.CycleState.Write(inflightAtDispatchKey, inflight) } updated[model] = true @@ -260,12 +256,11 @@ func (e *TTFTPercentileExtractor) Extract(ctx context.Context, events []dlsrc.Ev if s.Requests--; s.Requests < 0 { s.Requests = 0 } - reqID := p.Request.Headers["x-request-id"] if p.TTFT > 0 { ttft := p.TTFT.Seconds() var inflightAtDispatch int64 - if entry, found := s.pending[reqID]; found && reqID != "" { - inflightAtDispatch = entry.inflightAtDispatch + if p.CycleState != nil { + inflightAtDispatch, _ = plugin.ReadCycleStateKey[int64](p.CycleState, inflightAtDispatchKey) } s.tracker.add(ttft, inflightAtDispatch, now) if debugLogger.Enabled() { @@ -274,9 +269,6 @@ func (e *TTFTPercentileExtractor) Extract(ctx context.Context, events []dlsrc.Ev ) } } - if reqID != "" { - delete(s.pending, reqID) - } if now.Sub(s.intervalStart) >= e.intervalDuration { s.flush(now, e.maxObservationAge, e.lowLoadWindowAge, e.maxRequests) } @@ -286,12 +278,6 @@ func (e *TTFTPercentileExtractor) Extract(ctx context.Context, events []dlsrc.Ev for model := range updated { s := e.state[model] - cutoff := now.Add(-e.maxObservationAge) - for id, entry := range s.pending { - if entry.dispatchedAt.Before(cutoff) { - delete(s.pending, id) - } - } m := s.TTFTPercentileMetrics m.MinRequests = e.minRequests e.ds.GetOrCreateModel(model).GetAttributes().Put(AttributeKey, m) @@ -315,7 +301,6 @@ func (e *TTFTPercentileExtractor) getOrCreate(model string) *modelPercentileStat } s := &modelPercentileState{ tracker: newSlidingWindowTracker(e.windowSize), - pending: make(map[string]pendingEntry), } e.state[model] = s return s diff --git a/pkg/handlers/request.go b/pkg/handlers/request.go index a5a9a95d..04888960 100644 --- a/pkg/handlers/request.go +++ b/pkg/handlers/request.go @@ -94,7 +94,7 @@ func (s *Server) HandleRequestBody(ctx context.Context, reqCtx *RequestContext, // Notify the data layer of the incoming request after headers are fully formed. s.eventNotifier.Notify(datasource.Event{ Type: datasource.RequestEventType, - Payload: datasource.RequestPayload{Request: reqCtx.Request}, + Payload: datasource.RequestPayload{Request: reqCtx.Request, CycleState: reqCtx.CycleState}, }) metrics.RecordSuccessCounter() diff --git a/pkg/handlers/response.go b/pkg/handlers/response.go index 6836577d..5461f377 100644 --- a/pkg/handlers/response.go +++ b/pkg/handlers/response.go @@ -64,10 +64,11 @@ func (s *Server) HandleResponseBody(ctx context.Context, reqCtx *RequestContext, s.eventNotifier.Notify(datasource.Event{ Type: datasource.ResponseEventType, Payload: datasource.ResponsePayload{ - Request: reqCtx.Request, - Response: reqCtx.Response, - Duration: reqCtx.ResponseCompleteTimestamp.Sub(reqCtx.RequestReceivedTimestamp), - TTFT: ttft, + Request: reqCtx.Request, + Response: reqCtx.Response, + CycleState: reqCtx.CycleState, + Duration: reqCtx.ResponseCompleteTimestamp.Sub(reqCtx.RequestReceivedTimestamp), + TTFT: ttft, }, }) if ttft > 0 { From 07a7eff3095a8f1ab0f4c61f33d96e67b517acc4 Mon Sep 17 00:00:00 2001 From: Mohammad Date: Thu, 2 Jul 2026 09:17:12 +0300 Subject: [PATCH 15/27] Move the response plugin out. Signed-off-by: Mohammad --- cmd/runner/runner.go | 2 - examples/medianttft-values.yaml | 3 - .../interface/requesthandling/types.go | 3 - .../modelselector/scorer/queuettft/plugin.go | 54 +-------- .../requesthandling/modelselector/plugin.go | 6 +- .../scoringdebugheaders/plugin.go | 111 ------------------ pkg/handlers/response.go | 3 - 7 files changed, 5 insertions(+), 177 deletions(-) delete mode 100644 pkg/framework/plugins/responsehandling/scoringdebugheaders/plugin.go diff --git a/cmd/runner/runner.go b/cmd/runner/runner.go index 5d7e9697..1563a7af 100644 --- a/cmd/runner/runner.go +++ b/cmd/runner/runner.go @@ -56,7 +56,6 @@ import ( inflightrequestsscorer "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/plugins/modelselector/scorer/inflightrequests" queuettftscorer "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/plugins/modelselector/scorer/queuettft" "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/plugins/requesthandling/basemodelextractor" - scoringdebugheaders "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/plugins/responsehandling/scoringdebugheaders" "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/plugins/requesthandling/bodyfieldtoheader" modelselectorplugin "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/plugins/requesthandling/modelselector" "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/plugins/requesthandling/profilepicker/single" @@ -290,7 +289,6 @@ func (r *Runner) registerInTreePlugins() { plugin.Register(modelselectorplugin.ModelSelectorPluginType, modelselectorplugin.ModelSelectorPluginFactory) plugin.Register(inflightrequestsscorer.PluginType, inflightrequestsscorer.ScorerFactory) plugin.Register(queuettftscorer.PluginType, queuettftscorer.ScorerFactory) - plugin.Register(scoringdebugheaders.PluginType, scoringdebugheaders.Factory) } // registerHealthServer adds the Health gRPC server as a Runnable to the given manager. diff --git a/examples/medianttft-values.yaml b/examples/medianttft-values.yaml index 14b47844..fbc92ab6 100644 --- a/examples/medianttft-values.yaml +++ b/examples/medianttft-values.yaml @@ -24,7 +24,6 @@ payloadProcessor: lowLoadWindowAge: 1h maxRequests: 100 minRequests: 20 - - type: scoring-debug-headers - type: model-config-datasource parameters: modelsPath: /config/models.json @@ -38,8 +37,6 @@ payloadProcessor: - pluginRef: max-score-picker - pluginRef: body-field-to-header - pluginRef: base-model-to-header - response: - - pluginRef: scoring-debug-headers datalayer: extractors: - pluginRef: ttft-percentile-extractor diff --git a/pkg/framework/interface/requesthandling/types.go b/pkg/framework/interface/requesthandling/types.go index c79a7a77..78afa58d 100644 --- a/pkg/framework/interface/requesthandling/types.go +++ b/pkg/framework/interface/requesthandling/types.go @@ -22,9 +22,6 @@ import ( "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/interface/plugin" ) -// TTFTCycleStateKey holds the real TTFT (seconds, float64) measured by the handler. -const TTFTCycleStateKey = "ipp/ttft-seconds" - func newInferenceMessage() InferenceMessage { return InferenceMessage{ Headers: map[string]string{}, diff --git a/pkg/framework/plugins/modelselector/scorer/queuettft/plugin.go b/pkg/framework/plugins/modelselector/scorer/queuettft/plugin.go index 41592afc..8cf557be 100644 --- a/pkg/framework/plugins/modelselector/scorer/queuettft/plugin.go +++ b/pkg/framework/plugins/modelselector/scorer/queuettft/plugin.go @@ -42,22 +42,8 @@ const ( PluginType = "queue-ttft-scorer" defaultExplorationRate = 0.0 // off by default; set e.g. 0.1 for 10% exploration - - // DecisionsCycleStateKey holds the per-model scoring decisions for response plugins. - DecisionsCycleStateKey = "queue-ttft/decisions" ) -// ScoringDecision captures the scorer's decision for one model, written to CycleState. -type ScoringDecision struct { - EffectiveTTFT float64 - Floor float64 // P10Low - P50 float64 - RecentN int - Inflight int64 - State string // "trusted" | "seed" | "unobserved" - ExplorationSuppressed bool -} - var _ modelselector.Scorer = &QueueTTFTScorer{} // QueueTTFTScorerConfig holds optional parameters for the scorer plugin. @@ -107,11 +93,10 @@ func (s *QueueTTFTScorer) WithExplorationRate(r float64) *QueueTTFTScorer { // modelEval is the scorer's per-model working state for one Score call. type modelEval struct { - metrics ttftpercentile.TTFTPercentileMetrics - eff float64 - trusted bool // calibrated operating point - observed bool // has a service floor (not truly cold) - suppressed bool // exploration-suppressed this cycle + metrics ttftpercentile.TTFTPercentileMetrics + eff float64 + trusted bool // calibrated operating point + observed bool // has a service floor (not truly cold) } // Score ranks models by predicted TTFT: score = (maxTTFT − effectiveTTFT) / (maxTTFT − minTTFT). @@ -144,7 +129,6 @@ func (s *QueueTTFTScorer) Score(ctx context.Context, cycleState *plugin.CycleSta for _, model := range models { scores[model] = 1.0 } - s.writeDecisions(cycleState, models, evals) return scores } @@ -169,7 +153,6 @@ func (s *QueueTTFTScorer) Score(ctx context.Context, cycleState *plugin.CycleSta } if s.explorationRate > 0 && !e.trusted && rand.Float64() >= s.explorationRate { scores[model] = 0 - e.suppressed = true } } @@ -181,7 +164,6 @@ func (s *QueueTTFTScorer) Score(ctx context.Context, cycleState *plugin.CycleSta } } - s.writeDecisions(cycleState, models, evals) return scores } @@ -195,31 +177,3 @@ func metricsFor(model datalayer.Model) ttftpercentile.TTFTPercentileMetrics { } return ttftpercentile.TTFTPercentileMetrics{} } - -// writeDecisions stores per-model scoring decisions in CycleState for response plugins. -func (s *QueueTTFTScorer) writeDecisions(cycleState *plugin.CycleState, models []datalayer.Model, evals map[datalayer.Model]*modelEval) { - if cycleState == nil { - return - } - decisions := make(map[string]ScoringDecision, len(models)) - for _, model := range models { - e := evals[model] - state := "trusted" - switch { - case !e.observed: - state = "unobserved" - case !e.trusted: - state = "seed" - } - decisions[model.GetName()] = ScoringDecision{ - EffectiveTTFT: e.eff, - Floor: e.metrics.Floor(), - P50: e.metrics.P50TTFT, - RecentN: e.metrics.RecentN, - Inflight: e.metrics.Requests, - State: state, - ExplorationSuppressed: e.suppressed, - } - } - cycleState.Write(DecisionsCycleStateKey, decisions) -} diff --git a/pkg/framework/plugins/requesthandling/modelselector/plugin.go b/pkg/framework/plugins/requesthandling/modelselector/plugin.go index 168d74e1..b8ebf2aa 100644 --- a/pkg/framework/plugins/requesthandling/modelselector/plugin.go +++ b/pkg/framework/plugins/requesthandling/modelselector/plugin.go @@ -33,9 +33,6 @@ import ( const ( ModelSelectorPluginType = "model-selector" - - // SelectedModelCycleStateKey holds the selected model name for response plugins. - SelectedModelCycleStateKey = "model-selector/selected-model" ) var _ requesthandling.RequestProcessor = &ModelSelectorPlugin{} @@ -77,7 +74,7 @@ func (p *ModelSelectorPlugin) WithName(name string) *ModelSelectorPlugin { } // ProcessRequest reads candidate models from the Datastore, runs model -// selection, and writes the selected model into the request body and CycleState. +// selection, and writes the selected model into the request body. func (p *ModelSelectorPlugin) ProcessRequest(ctx context.Context, cycleState *plugin.CycleState, request *requesthandling.InferenceRequest) error { logger := log.FromContext(ctx) @@ -94,7 +91,6 @@ func (p *ModelSelectorPlugin) ProcessRequest(ctx context.Context, cycleState *pl selectedName := result.TargetModel.GetName() logger.V(logutil.VERBOSE).Info("Model selected", "model", selectedName) - cycleState.Write(SelectedModelCycleStateKey, selectedName) request.SetBodyField("model", selectedName) return nil diff --git a/pkg/framework/plugins/responsehandling/scoringdebugheaders/plugin.go b/pkg/framework/plugins/responsehandling/scoringdebugheaders/plugin.go deleted file mode 100644 index be38bf23..00000000 --- a/pkg/framework/plugins/responsehandling/scoringdebugheaders/plugin.go +++ /dev/null @@ -1,111 +0,0 @@ -/* -Copyright 2026 The llm-d Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package scoringdebugheaders writes queue-ttft scoring decisions as response headers. -package scoringdebugheaders - -import ( - "context" - "encoding/json" - "fmt" - - "sigs.k8s.io/controller-runtime/pkg/log" - - logutil "github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/logging" - "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/interface/plugin" - "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/interface/requesthandling" - queuettft "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/plugins/modelselector/scorer/queuettft" - modelselector "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/plugins/requesthandling/modelselector" -) - -const ( - PluginType = "scoring-debug-headers" - - headerModel = "X-IPP-Scorer-Model" - headerState = "X-IPP-Scorer-State" - headerEffTTFT = "X-IPP-Scorer-EffectiveTTFT" - headerRealTTFT = "X-IPP-Real-TTFT" - headerFloor = "X-IPP-Scorer-Floor" - headerP50 = "X-IPP-Scorer-P50" - headerRecentN = "X-IPP-Scorer-RecentN" - headerInflight = "X-IPP-Scorer-Inflight" - headerExploration = "X-IPP-Scorer-ExplorationSuppressed" -) - -var _ requesthandling.ResponseProcessor = &ScoringDebugHeadersPlugin{} - -func Factory(name string, _ json.RawMessage, _ plugin.Handle) (plugin.Plugin, error) { - return NewScoringDebugHeadersPlugin().WithName(name), nil -} - -func NewScoringDebugHeadersPlugin() *ScoringDebugHeadersPlugin { - return &ScoringDebugHeadersPlugin{ - typedName: plugin.TypedName{Type: PluginType, Name: PluginType}, - } -} - -type ScoringDebugHeadersPlugin struct { - typedName plugin.TypedName -} - -func (p *ScoringDebugHeadersPlugin) TypedName() plugin.TypedName { return p.typedName } -func (p *ScoringDebugHeadersPlugin) WithName(name string) *ScoringDebugHeadersPlugin { - p.typedName.Name = name; return p -} - -func (p *ScoringDebugHeadersPlugin) ProcessResponse(ctx context.Context, cycleState *plugin.CycleState, response *requesthandling.InferenceResponse) error { - selectedModel, err := plugin.ReadCycleStateKey[string](cycleState, modelselector.SelectedModelCycleStateKey) - if err != nil { - log.FromContext(ctx).V(logutil.DEBUG).Info("scoring-debug-headers: no selected model in CycleState, skipping") - return nil - } - - decisions, err := plugin.ReadCycleStateKey[map[string]queuettft.ScoringDecision](cycleState, queuettft.DecisionsCycleStateKey) - if err != nil { - log.FromContext(ctx).V(logutil.DEBUG).Info("scoring-debug-headers: no scoring decisions in CycleState, skipping") - return nil - } - - d, ok := decisions[selectedModel] - if !ok { - log.FromContext(ctx).V(logutil.DEBUG).Info("scoring-debug-headers: selected model not in decisions", "model", selectedModel) - return nil - } - - response.SetHeader(headerModel, selectedModel) - response.SetHeader(headerState, d.State) - response.SetHeader(headerRecentN, fmt.Sprintf("%d", d.RecentN)) - response.SetHeader(headerInflight, fmt.Sprintf("%d", d.Inflight)) - - if d.Floor > 0 { - response.SetHeader(headerFloor, fmt.Sprintf("%.3f", d.Floor)) - } - if d.P50 > 0 { - response.SetHeader(headerP50, fmt.Sprintf("%.3f", d.P50)) - } - if d.EffectiveTTFT > 0 { - response.SetHeader(headerEffTTFT, fmt.Sprintf("%.3f", d.EffectiveTTFT)) - } - if d.ExplorationSuppressed { - response.SetHeader(headerExploration, "true") - } - - if realTTFT, err := plugin.ReadCycleStateKey[float64](cycleState, requesthandling.TTFTCycleStateKey); err == nil && realTTFT > 0 { - response.SetHeader(headerRealTTFT, fmt.Sprintf("%.3f", realTTFT)) - } - - return nil -} diff --git a/pkg/handlers/response.go b/pkg/handlers/response.go index 5461f377..96816a48 100644 --- a/pkg/handlers/response.go +++ b/pkg/handlers/response.go @@ -71,9 +71,6 @@ func (s *Server) HandleResponseBody(ctx context.Context, reqCtx *RequestContext, TTFT: ttft, }, }) - if ttft > 0 { - reqCtx.CycleState.Write(requesthandling.TTFTCycleStateKey, ttft.Seconds()) - } logger := log.FromContext(ctx) if len(reqCtx.Profile.ResponsePlugins) == 0 { From 30851801d1259467e36461f311478118108c4a04 Mon Sep 17 00:00:00 2001 From: Mohammad Date: Thu, 2 Jul 2026 11:16:35 +0300 Subject: [PATCH 16/27] Separate p10low interval from p50 interval. Signed-off-by: Mohammad --- .../datalayer/ttftpercentile/plugin.go | 36 +++++++++++++++---- .../modelselector/scorer/queuettft/README.md | 1 + 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/pkg/framework/plugins/datalayer/ttftpercentile/plugin.go b/pkg/framework/plugins/datalayer/ttftpercentile/plugin.go index 55a15c7e..0623cb6c 100644 --- a/pkg/framework/plugins/datalayer/ttftpercentile/plugin.go +++ b/pkg/framework/plugins/datalayer/ttftpercentile/plugin.go @@ -43,6 +43,7 @@ const ( defaultWindowSize = 5000 defaultMaxObservationAge = 3 * time.Minute // observations older than this are never used defaultIntervalDuration = 5 * time.Second + defaultFloorInterval = 1 * time.Minute // P10Low recompute cadence (slow-moving floor) defaultLowLoadWindowAge = 1 * time.Hour defaultMaxRequests = 100 // cap the short window to the most recent N observations defaultMinRequests = 10 // below this count the scorer falls back to the optimistic seed @@ -52,7 +53,9 @@ var _ dlsrc.Extractor = &TTFTPercentileExtractor{} type TTFTPercentileExtractorConfig struct { IntervalDuration string `json:"intervalDuration,omitempty"` - WindowSize int `json:"windowSize,omitempty"` + // FloorInterval is how often the P10Low floor is recomputed. Defaults to 1m. + FloorInterval string `json:"floorInterval,omitempty"` + WindowSize int `json:"windowSize,omitempty"` // MaxObservationAge caps how far back the short window looks. // Observations older than this are never used for P50 or short-window P10. MaxObservationAge string `json:"maxObservationAge,omitempty"` @@ -107,11 +110,13 @@ func (m TTFTPercentileMetrics) Predict() (effectiveTTFT float64, trusted bool) { type modelPercentileState struct { TTFTPercentileMetrics intervalStart time.Time + floorStart time.Time tracker *slidingWindowTracker } -func (s *modelPercentileState) flush(now time.Time, maxObservationAge, lowLoadWindowAge time.Duration, maxRequests int) { +func (s *modelPercentileState) flush(now time.Time, maxObservationAge, lowLoadWindowAge, floorInterval time.Duration, maxRequests int) { // Short window (value-sorted, capped): one snapshot feeds P10, P50 and inflightAtP50. + // Recomputed every interval to keep the operating point fresh. short := s.tracker.window(now, maxObservationAge, maxRequests) s.RecentN = len(short) if len(short) > 0 { @@ -120,11 +125,16 @@ func (s *modelPercentileState) flush(now time.Time, maxObservationAge, lowLoadWi s.InflightAtP50 = bandInflight(short, 0.50) s.LastObservedAt = now.UnixNano() } - // Long window (uncapped): stable hardware floor. - if long := s.tracker.window(now, lowLoadWindowAge, 0); len(long) > 0 { - s.P10LowTTFT = twoLevelP10(long) - } s.intervalStart = now + + // Long window (uncapped): stable hardware floor. Recomputed at most once per floorInterval, + // since its scan+sort is the expensive part of flush and the floor barely changes. + if now.Sub(s.floorStart) >= floorInterval { + if long := s.tracker.window(now, lowLoadWindowAge, 0); len(long) > 0 { + s.P10LowTTFT = twoLevelP10(long) + } + s.floorStart = now + } } type TTFTPercentileExtractor struct { @@ -134,6 +144,7 @@ type TTFTPercentileExtractor struct { windowSize int maxObservationAge time.Duration intervalDuration time.Duration + floorInterval time.Duration lowLoadWindowAge time.Duration maxRequests int minRequests int @@ -142,6 +153,7 @@ type TTFTPercentileExtractor struct { func ExtractorFactory(name string, parameters json.RawMessage, h plugin.Handle) (plugin.Plugin, error) { cfg := TTFTPercentileExtractorConfig{ IntervalDuration: defaultIntervalDuration.String(), + FloorInterval: defaultFloorInterval.String(), WindowSize: defaultWindowSize, MaxObservationAge: defaultMaxObservationAge.String(), LowLoadWindowAge: defaultLowLoadWindowAge.String(), @@ -166,6 +178,10 @@ func ExtractorFactory(name string, parameters json.RawMessage, h plugin.Handle) if err != nil { return nil, fmt.Errorf("invalid intervalDuration %q for plugin %q: %w", cfg.IntervalDuration, name, err) } + floorInterval, err := time.ParseDuration(cfg.FloorInterval) + if err != nil { + return nil, fmt.Errorf("invalid floorInterval %q for plugin %q: %w", cfg.FloorInterval, name, err) + } maxObsAge, err := time.ParseDuration(cfg.MaxObservationAge) if err != nil { return nil, fmt.Errorf("invalid maxObservationAge %q for plugin %q: %w", cfg.MaxObservationAge, name, err) @@ -177,6 +193,7 @@ func ExtractorFactory(name string, parameters json.RawMessage, h plugin.Handle) return NewTTFTPercentileExtractor(h.Datastore()). WithName(name). WithIntervalDuration(interval). + WithFloorInterval(floorInterval). WithWindow(cfg.WindowSize, maxObsAge). WithLowLoadWindowAge(lowLoadAge). WithRequestBounds(cfg.MaxRequests, cfg.MinRequests), nil @@ -190,6 +207,7 @@ func NewTTFTPercentileExtractor(ds datalayer.Datastore) *TTFTPercentileExtractor windowSize: defaultWindowSize, maxObservationAge: defaultMaxObservationAge, intervalDuration: defaultIntervalDuration, + floorInterval: defaultFloorInterval, lowLoadWindowAge: defaultLowLoadWindowAge, maxRequests: defaultMaxRequests, minRequests: defaultMinRequests, @@ -205,6 +223,10 @@ func (e *TTFTPercentileExtractor) WithIntervalDuration(d time.Duration) *TTFTPer e.intervalDuration = d return e } +func (e *TTFTPercentileExtractor) WithFloorInterval(d time.Duration) *TTFTPercentileExtractor { + e.floorInterval = d + return e +} func (e *TTFTPercentileExtractor) WithWindow(size int, maxObsAge time.Duration) *TTFTPercentileExtractor { e.windowSize, e.maxObservationAge = size, maxObsAge return e @@ -270,7 +292,7 @@ func (e *TTFTPercentileExtractor) Extract(ctx context.Context, events []dlsrc.Ev } } if now.Sub(s.intervalStart) >= e.intervalDuration { - s.flush(now, e.maxObservationAge, e.lowLoadWindowAge, e.maxRequests) + s.flush(now, e.maxObservationAge, e.lowLoadWindowAge, e.floorInterval, e.maxRequests) } updated[model] = true } diff --git a/pkg/framework/plugins/modelselector/scorer/queuettft/README.md b/pkg/framework/plugins/modelselector/scorer/queuettft/README.md index 5191ec8d..7941b855 100644 --- a/pkg/framework/plugins/modelselector/scorer/queuettft/README.md +++ b/pkg/framework/plugins/modelselector/scorer/queuettft/README.md @@ -69,6 +69,7 @@ will wait. No fitting, no tunable parameters — just two observed points. | `maxRequests` | 100 | Cap the short window to the most recent N observations | | `minRequests` | 10 | Minimum capped-window count before the scorer trusts the formula | | `lowLoadWindowAge` | 1h | Window for two-level P10Low (long = stable hardware floor) | +| `floorInterval` | 1m | How often P10Low is recomputed (slow-moving floor; cheaper than every interval) | | `windowSize` | 5000 | Ring buffer capacity (~200 KB per model) | ## Possible Enhancements From d3841d6a6a2c9724cdf30595046065f0e3c4451e Mon Sep 17 00:00:00 2001 From: Mohammad Date: Thu, 2 Jul 2026 12:13:00 +0300 Subject: [PATCH 17/27] Small fix. Signed-off-by: Mohammad --- pkg/framework/plugins/requesthandling/modelselector/plugin.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/framework/plugins/requesthandling/modelselector/plugin.go b/pkg/framework/plugins/requesthandling/modelselector/plugin.go index b8ebf2aa..21168037 100644 --- a/pkg/framework/plugins/requesthandling/modelselector/plugin.go +++ b/pkg/framework/plugins/requesthandling/modelselector/plugin.go @@ -74,7 +74,7 @@ func (p *ModelSelectorPlugin) WithName(name string) *ModelSelectorPlugin { } // ProcessRequest reads candidate models from the Datastore, runs model -// selection, and writes the selected model into the request body. +// selection, and writes the selected model into the request body and CycleState. func (p *ModelSelectorPlugin) ProcessRequest(ctx context.Context, cycleState *plugin.CycleState, request *requesthandling.InferenceRequest) error { logger := log.FromContext(ctx) From 30dd1ddd3aed35a70dc0ec35f2293db3460e66d2 Mon Sep 17 00:00:00 2001 From: Mohammad Date: Thu, 2 Jul 2026 13:20:47 +0300 Subject: [PATCH 18/27] Split the PR. Signed-off-by: Mohammad --- cmd/runner/runner.go | 2 - .../interface/datalayer/datasource/types.go | 12 +- .../datalayer/ttftpercentile/plugin.go | 329 ------------------ .../datalayer/ttftpercentile/tracker.go | 108 ------ pkg/handlers/request.go | 2 +- pkg/handlers/response.go | 10 +- pkg/handlers/server.go | 2 +- 7 files changed, 11 insertions(+), 454 deletions(-) delete mode 100644 pkg/framework/plugins/datalayer/ttftpercentile/plugin.go delete mode 100644 pkg/framework/plugins/datalayer/ttftpercentile/tracker.go diff --git a/cmd/runner/runner.go b/cmd/runner/runner.go index f602f34d..4d428a9b 100644 --- a/cmd/runner/runner.go +++ b/cmd/runner/runner.go @@ -49,7 +49,6 @@ import ( "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/interface/requesthandling" modelconfigcollector "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/plugins/datalayer/modelconfigcollector" requestmetadata "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/plugins/datalayer/requestmetadata" - "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/plugins/datalayer/ttftpercentile" "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/plugins/modelselector/picker/maxscore" "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/plugins/modelselector/picker/random" "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/plugins/modelselector/picker/weightedrandom" @@ -290,7 +289,6 @@ func (r *Runner) registerInTreePlugins() { plugin.Register(basemodelextractor.BaseModelToHeaderPluginType, basemodelextractor.BaseModelToHeaderPluginFactory) plugin.Register(requestmetadata.PluginType, requestmetadata.ExtractorFactory) plugin.Register(modelconfigcollector.PluginType, modelconfigcollector.DatasourceFactory) - plugin.Register(ttftpercentile.PluginType, ttftpercentile.ExtractorFactory) // register model selector plugins plugin.Register(random.RandomPickerType, random.RandomPickerFactory) plugin.Register(maxscore.MaxScorePickerType, maxscore.MaxScorePickerFactory) diff --git a/pkg/framework/interface/datalayer/datasource/types.go b/pkg/framework/interface/datalayer/datasource/types.go index b22f36f3..016605ce 100644 --- a/pkg/framework/interface/datalayer/datasource/types.go +++ b/pkg/framework/interface/datalayer/datasource/types.go @@ -46,17 +46,15 @@ const ( // RequestPayload is the Payload for RequestEventType. type RequestPayload struct { - Request *requesthandling.InferenceRequest - CycleState *plugin.CycleState + Request *requesthandling.InferenceRequest } // ResponsePayload is the Payload for ResponseEventType. type ResponsePayload struct { - Request *requesthandling.InferenceRequest - Response *requesthandling.InferenceResponse - CycleState *plugin.CycleState - Duration time.Duration - TTFT time.Duration + Request *requesthandling.InferenceRequest + Response *requesthandling.InferenceResponse + Duration time.Duration + TTFT time.Duration } type DatalayerProcessor interface { diff --git a/pkg/framework/plugins/datalayer/ttftpercentile/plugin.go b/pkg/framework/plugins/datalayer/ttftpercentile/plugin.go deleted file mode 100644 index 0623cb6c..00000000 --- a/pkg/framework/plugins/datalayer/ttftpercentile/plugin.go +++ /dev/null @@ -1,329 +0,0 @@ -/* -Copyright 2026 The llm-d Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package ttftpercentile tracks per-model TTFT distributions and publishes -// P10Low, P50, and inflightAtP50 for the queue-ttft-scorer. -package ttftpercentile - -import ( - "context" - "encoding/json" - "fmt" - "time" - - "sigs.k8s.io/controller-runtime/pkg/log" - - logutil "github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/logging" - "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/interface/datalayer" - dlsrc "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/interface/datalayer/datasource" - "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/interface/plugin" -) - -const ( - PluginType = "ttft-percentile-extractor" - AttributeKey = "ttft-percentile" - - // inflightAtDispatchKey carries a request's inflight-at-dispatch through CycleState - // from the request event to the matching response event. - inflightAtDispatchKey = "ttft-percentile/inflight-at-dispatch" - - defaultWindowSize = 5000 - defaultMaxObservationAge = 3 * time.Minute // observations older than this are never used - defaultIntervalDuration = 5 * time.Second - defaultFloorInterval = 1 * time.Minute // P10Low recompute cadence (slow-moving floor) - defaultLowLoadWindowAge = 1 * time.Hour - defaultMaxRequests = 100 // cap the short window to the most recent N observations - defaultMinRequests = 10 // below this count the scorer falls back to the optimistic seed -) - -var _ dlsrc.Extractor = &TTFTPercentileExtractor{} - -type TTFTPercentileExtractorConfig struct { - IntervalDuration string `json:"intervalDuration,omitempty"` - // FloorInterval is how often the P10Low floor is recomputed. Defaults to 1m. - FloorInterval string `json:"floorInterval,omitempty"` - WindowSize int `json:"windowSize,omitempty"` - // MaxObservationAge caps how far back the short window looks. - // Observations older than this are never used for P50 or short-window P10. - MaxObservationAge string `json:"maxObservationAge,omitempty"` - LowLoadWindowAge string `json:"lowLoadWindowAge,omitempty"` - // MaxRequests caps the short window to the most recent N observations regardless of age. - MaxRequests int `json:"maxRequests,omitempty"` - // MinRequests is the minimum capped-window count for the scorer to use the trusted - // operating point. Below this the scorer falls back to the optimistic seed (floor only). - MinRequests int `json:"minRequests,omitempty"` -} - -// TTFTPercentileMetrics is written to each model's attribute store every intervalDuration. -type TTFTPercentileMetrics struct { - Requests int64 - InflightAtP50 float64 // avg inflight_at_dispatch of observations in the P40-P60 band - P10LowTTFT float64 // two-level P10: P10 of the bottom decile; hardware-floor estimate - P10TTFT float64 // P10 from capped short window - P50TTFT float64 // P50 from capped short window - LastObservedAt int64 - RecentN int // count of observations in the capped short window - MinRequests int // scorer threshold — copied from config so the scorer needs no separate param -} - -func (m TTFTPercentileMetrics) Clone() datalayer.Cloneable { return m } - -// Floor is the load-invariant service floor: P10Low, or P10 before the long window fills. -// Zero means the model is truly cold. -func (m TTFTPercentileMetrics) Floor() float64 { - if m.P10LowTTFT > 0 { - return m.P10LowTTFT - } - return m.P10TTFT -} - -// Predict returns the effective TTFT at the current inflight and whether it is trusted -// (calibrated). Uncalibrated but observed → floor as an optimistic seed; cold → (0, false). -func (m TTFTPercentileMetrics) Predict() (effectiveTTFT float64, trusted bool) { - floor := m.Floor() - if floor == 0 { - return 0, false - } - if m.RecentN >= m.MinRequests && m.InflightAtP50 > 0 && m.P50TTFT > floor { - eff := floor + float64(m.Requests)*(m.P50TTFT-floor)/m.InflightAtP50 - if eff < floor { - eff = floor - } - return eff, true - } - return floor, false -} - -type modelPercentileState struct { - TTFTPercentileMetrics - intervalStart time.Time - floorStart time.Time - tracker *slidingWindowTracker -} - -func (s *modelPercentileState) flush(now time.Time, maxObservationAge, lowLoadWindowAge, floorInterval time.Duration, maxRequests int) { - // Short window (value-sorted, capped): one snapshot feeds P10, P50 and inflightAtP50. - // Recomputed every interval to keep the operating point fresh. - short := s.tracker.window(now, maxObservationAge, maxRequests) - s.RecentN = len(short) - if len(short) > 0 { - s.P10TTFT = percentileOf(short, 0.10) - s.P50TTFT = percentileOf(short, 0.50) - s.InflightAtP50 = bandInflight(short, 0.50) - s.LastObservedAt = now.UnixNano() - } - s.intervalStart = now - - // Long window (uncapped): stable hardware floor. Recomputed at most once per floorInterval, - // since its scan+sort is the expensive part of flush and the floor barely changes. - if now.Sub(s.floorStart) >= floorInterval { - if long := s.tracker.window(now, lowLoadWindowAge, 0); len(long) > 0 { - s.P10LowTTFT = twoLevelP10(long) - } - s.floorStart = now - } -} - -type TTFTPercentileExtractor struct { - typedName plugin.TypedName - ds datalayer.Datastore - state map[string]*modelPercentileState - windowSize int - maxObservationAge time.Duration - intervalDuration time.Duration - floorInterval time.Duration - lowLoadWindowAge time.Duration - maxRequests int - minRequests int -} - -func ExtractorFactory(name string, parameters json.RawMessage, h plugin.Handle) (plugin.Plugin, error) { - cfg := TTFTPercentileExtractorConfig{ - IntervalDuration: defaultIntervalDuration.String(), - FloorInterval: defaultFloorInterval.String(), - WindowSize: defaultWindowSize, - MaxObservationAge: defaultMaxObservationAge.String(), - LowLoadWindowAge: defaultLowLoadWindowAge.String(), - MaxRequests: defaultMaxRequests, - MinRequests: defaultMinRequests, - } - if len(parameters) > 0 { - if err := json.Unmarshal(parameters, &cfg); err != nil { - return nil, fmt.Errorf("failed to parse parameters for plugin %q: %w", name, err) - } - } - if cfg.WindowSize <= 0 { - return nil, fmt.Errorf("windowSize must be > 0 for plugin %q", name) - } - if cfg.MaxRequests <= 0 { - return nil, fmt.Errorf("maxRequests must be > 0 for plugin %q", name) - } - if cfg.MinRequests <= 0 { - return nil, fmt.Errorf("minRequests must be > 0 for plugin %q", name) - } - interval, err := time.ParseDuration(cfg.IntervalDuration) - if err != nil { - return nil, fmt.Errorf("invalid intervalDuration %q for plugin %q: %w", cfg.IntervalDuration, name, err) - } - floorInterval, err := time.ParseDuration(cfg.FloorInterval) - if err != nil { - return nil, fmt.Errorf("invalid floorInterval %q for plugin %q: %w", cfg.FloorInterval, name, err) - } - maxObsAge, err := time.ParseDuration(cfg.MaxObservationAge) - if err != nil { - return nil, fmt.Errorf("invalid maxObservationAge %q for plugin %q: %w", cfg.MaxObservationAge, name, err) - } - lowLoadAge, err := time.ParseDuration(cfg.LowLoadWindowAge) - if err != nil { - return nil, fmt.Errorf("invalid lowLoadWindowAge %q for plugin %q: %w", cfg.LowLoadWindowAge, name, err) - } - return NewTTFTPercentileExtractor(h.Datastore()). - WithName(name). - WithIntervalDuration(interval). - WithFloorInterval(floorInterval). - WithWindow(cfg.WindowSize, maxObsAge). - WithLowLoadWindowAge(lowLoadAge). - WithRequestBounds(cfg.MaxRequests, cfg.MinRequests), nil -} - -func NewTTFTPercentileExtractor(ds datalayer.Datastore) *TTFTPercentileExtractor { - return &TTFTPercentileExtractor{ - typedName: plugin.TypedName{Type: PluginType, Name: PluginType}, - ds: ds, - state: make(map[string]*modelPercentileState), - windowSize: defaultWindowSize, - maxObservationAge: defaultMaxObservationAge, - intervalDuration: defaultIntervalDuration, - floorInterval: defaultFloorInterval, - lowLoadWindowAge: defaultLowLoadWindowAge, - maxRequests: defaultMaxRequests, - minRequests: defaultMinRequests, - } -} - -func (e *TTFTPercentileExtractor) TypedName() plugin.TypedName { return e.typedName } -func (e *TTFTPercentileExtractor) WithName(n string) *TTFTPercentileExtractor { - e.typedName.Name = n - return e -} -func (e *TTFTPercentileExtractor) WithIntervalDuration(d time.Duration) *TTFTPercentileExtractor { - e.intervalDuration = d - return e -} -func (e *TTFTPercentileExtractor) WithFloorInterval(d time.Duration) *TTFTPercentileExtractor { - e.floorInterval = d - return e -} -func (e *TTFTPercentileExtractor) WithWindow(size int, maxObsAge time.Duration) *TTFTPercentileExtractor { - e.windowSize, e.maxObservationAge = size, maxObsAge - return e -} -func (e *TTFTPercentileExtractor) WithLowLoadWindowAge(age time.Duration) *TTFTPercentileExtractor { - e.lowLoadWindowAge = age - return e -} -func (e *TTFTPercentileExtractor) WithRequestBounds(maxN, minN int) *TTFTPercentileExtractor { - e.maxRequests, e.minRequests = maxN, minN - return e -} - -func (e *TTFTPercentileExtractor) Extract(ctx context.Context, events []dlsrc.Event) error { - debugLogger := log.FromContext(ctx).V(logutil.DEBUG) - now := time.Now() - updated := map[string]bool{} - - for _, ev := range events { - switch ev.Type { - case dlsrc.RequestEventType: - p, ok := ev.Payload.(dlsrc.RequestPayload) - if !ok { - continue - } - model, _ := p.Request.Body["model"].(string) - if model == "" { - continue - } - s := e.getOrCreate(model) - inflight := s.Requests - s.Requests++ - // Stash inflight-at-dispatch in CycleState; the matching response event reads it back. - if p.CycleState != nil { - p.CycleState.Write(inflightAtDispatchKey, inflight) - } - updated[model] = true - - case dlsrc.ResponseEventType: - p, ok := ev.Payload.(dlsrc.ResponsePayload) - if !ok { - continue - } - model, _ := p.Request.Body["model"].(string) - if model == "" { - continue - } - s := e.getOrCreate(model) - if s.Requests--; s.Requests < 0 { - s.Requests = 0 - } - if p.TTFT > 0 { - ttft := p.TTFT.Seconds() - var inflightAtDispatch int64 - if p.CycleState != nil { - inflightAtDispatch, _ = plugin.ReadCycleStateKey[int64](p.CycleState, inflightAtDispatchKey) - } - s.tracker.add(ttft, inflightAtDispatch, now) - if debugLogger.Enabled() { - debugLogger.Info("ttft-observation", - "model", model, "ttft_s", ttft, "inflightAtDispatch", inflightAtDispatch, - ) - } - } - if now.Sub(s.intervalStart) >= e.intervalDuration { - s.flush(now, e.maxObservationAge, e.lowLoadWindowAge, e.floorInterval, e.maxRequests) - } - updated[model] = true - } - } - - for model := range updated { - s := e.state[model] - m := s.TTFTPercentileMetrics - m.MinRequests = e.minRequests - e.ds.GetOrCreateModel(model).GetAttributes().Put(AttributeKey, m) - - if debugLogger.Enabled() { - eff, _ := m.Predict() - debugLogger.Info("ttft-percentile wrote attribute", - "model", model, "Requests", m.Requests, - "InflightAtP50", m.InflightAtP50, "P10Low_s", m.P10LowTTFT, - "P10_s", m.P10TTFT, "P50_s", m.P50TTFT, "EffectiveTTFT_s", eff, - "RecentN", m.RecentN, "MinRequests", m.MinRequests, - ) - } - } - return nil -} - -func (e *TTFTPercentileExtractor) getOrCreate(model string) *modelPercentileState { - if s, ok := e.state[model]; ok { - return s - } - s := &modelPercentileState{ - tracker: newSlidingWindowTracker(e.windowSize), - } - e.state[model] = s - return s -} diff --git a/pkg/framework/plugins/datalayer/ttftpercentile/tracker.go b/pkg/framework/plugins/datalayer/ttftpercentile/tracker.go deleted file mode 100644 index fa7a31cb..00000000 --- a/pkg/framework/plugins/datalayer/ttftpercentile/tracker.go +++ /dev/null @@ -1,108 +0,0 @@ -/* -Copyright 2026 The llm-d Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package ttftpercentile - -import ( - "sort" - "time" -) - -type observation struct { - value float64 - inflight int64 // inflight_at_dispatch - ts time.Time -} - -// slidingWindowTracker is a fixed-capacity ring buffer of TTFT observations. -type slidingWindowTracker struct { - buf []observation - head int - n int -} - -func newSlidingWindowTracker(capacity int) *slidingWindowTracker { - return &slidingWindowTracker{buf: make([]observation, capacity)} -} - -func (t *slidingWindowTracker) add(value float64, inflight int64, ts time.Time) { - t.buf[t.head] = observation{value: value, inflight: inflight, ts: ts} - t.head = (t.head + 1) % len(t.buf) - if t.n < len(t.buf) { - t.n++ - } -} - -// window returns the observations within maxAge, capped to the most recent maxN -// (0 = no cap), sorted ascending by value. Observations are stored in time order, -// so the newest-first scan stops at the first one older than the cutoff. -func (t *slidingWindowTracker) window(now time.Time, maxAge time.Duration, maxN int) []observation { - cutoff := now.Add(-maxAge) - size := len(t.buf) - capHint := t.n - if maxN > 0 && maxN < capHint { - capHint = maxN - } - out := make([]observation, 0, capHint) - for i := 0; i < t.n; i++ { - o := t.buf[(t.head-1-i+size)%size] - if o.ts.Before(cutoff) { - break - } - out = append(out, o) - if maxN > 0 && len(out) == maxN { - break - } - } - sort.Slice(out, func(i, j int) bool { return out[i].value < out[j].value }) - return out -} - -// percentileOf returns the p-th percentile value of a value-sorted slice by linear interpolation. -func percentileOf(sorted []observation, p float64) float64 { - idx := p * float64(len(sorted)-1) - lo := int(idx) - if lo+1 >= len(sorted) { - return sorted[lo].value - } - return sorted[lo].value + (idx-float64(lo))*(sorted[lo+1].value-sorted[lo].value) -} - -// twoLevelP10 estimates the hardware floor: the P10 of the bottom-decile slice (~P1 of all), -// isolating the fastest requests in the window regardless of their inflight count. -func twoLevelP10(sorted []observation) float64 { - lo := int(0.10 * float64(len(sorted)-1)) - return percentileOf(sorted[:lo+1], 0.10) -} - -// bandInflight returns the average inflight of observations in the [p-0.1, p+0.1] band of -// a value-sorted slice. Averaging a band rather than a single point stabilises the estimate. -func bandInflight(sorted []observation, p float64) float64 { - n := len(sorted) - lo := int((p - 0.10) * float64(n-1)) - if lo < 0 { - lo = 0 - } - hi := int((p + 0.10) * float64(n-1)) - if hi >= n { - hi = n - 1 - } - var sum float64 - for i := lo; i <= hi; i++ { - sum += float64(sorted[i].inflight) - } - return sum / float64(hi-lo+1) -} diff --git a/pkg/handlers/request.go b/pkg/handlers/request.go index 0cd17a63..4203299e 100644 --- a/pkg/handlers/request.go +++ b/pkg/handlers/request.go @@ -98,7 +98,7 @@ func (s *Server) HandleRequestBody(ctx context.Context, reqCtx *RequestContext, // Notify the data layer of the incoming request after headers are fully formed. s.eventNotifier.Notify(datasource.Event{ Type: datasource.RequestEventType, - Payload: datasource.RequestPayload{Request: reqCtx.Request, CycleState: reqCtx.CycleState}, + Payload: datasource.RequestPayload{Request: reqCtx.Request}, }) metrics.RecordSuccessCounter() diff --git a/pkg/handlers/response.go b/pkg/handlers/response.go index f43236d9..4a674fbd 100644 --- a/pkg/handlers/response.go +++ b/pkg/handlers/response.go @@ -60,15 +60,13 @@ func (s *Server) HandleResponseHeaders(ctx context.Context, reqCtx *RequestConte // HandleResponseBody handles response bodies by executing response plugins in order. func (s *Server) HandleResponseBody(ctx context.Context, reqCtx *RequestContext, responseBodyBytes []byte) ([]*eppb.ProcessingResponse, error) { // Notify the data layer of the completed response. - ttft := reqCtx.ResponseFirstChunkTimestamp.Sub(reqCtx.RequestSentTimestamp) s.eventNotifier.Notify(datasource.Event{ Type: datasource.ResponseEventType, Payload: datasource.ResponsePayload{ - Request: reqCtx.Request, - Response: reqCtx.Response, - CycleState: reqCtx.CycleState, - Duration: reqCtx.ResponseCompleteTimestamp.Sub(reqCtx.RequestReceivedTimestamp), - TTFT: ttft, + Request: reqCtx.Request, + Response: reqCtx.Response, + Duration: reqCtx.ResponseCompleteTimestamp.Sub(reqCtx.RequestReceivedTimestamp), + TTFT: reqCtx.ResponseFirstChunkTimestamp.Sub(reqCtx.RequestSentTimestamp), }, }) diff --git a/pkg/handlers/server.go b/pkg/handlers/server.go index 2a10f1a6..89e4e1d6 100644 --- a/pkg/handlers/server.go +++ b/pkg/handlers/server.go @@ -179,7 +179,7 @@ func (s *Server) Process(srv extProcPb.ExternalProcessor_ProcessServer) error { loggerVerbose.Info("processing response headers complete") case *extProcPb.ProcessingRequest_ResponseBody: // Logged at TRACE: one line per stream chunk is a firehose under load (≈36 lines/request) - // that drowns scorer/extractor DEBUG logs and triggers kubelet log rotation. Use --v=5 to see it. + // that drowns scorer/extractor DEBUG logs. Use --v=5 to see it. logger.V(logutil.TRACE).Info("Incoming response body chunk", "EoS", v.ResponseBody.EndOfStream) if reqCtx.ResponseFirstChunkTimestamp.IsZero() { reqCtx.ResponseFirstChunkTimestamp = time.Now() From e181d82dd6c89be9c33072d8ed6834ab1515c82b Mon Sep 17 00:00:00 2001 From: Mohammad Date: Thu, 2 Jul 2026 13:43:30 +0300 Subject: [PATCH 19/27] Rename to load-aware-ttft-scorer. Signed-off-by: Mohammad --- cmd/runner/runner.go | 4 +-- .../payload-processor/templates/config.yaml | 7 ---- examples/medianttft-values.yaml | 4 +-- .../{queuettft => loadawarettft}/README.md | 4 +-- .../{queuettft => loadawarettft}/plugin.go | 32 +++++++++---------- 5 files changed, 22 insertions(+), 29 deletions(-) rename pkg/framework/plugins/modelselector/scorer/{queuettft => loadawarettft}/README.md (98%) rename pkg/framework/plugins/modelselector/scorer/{queuettft => loadawarettft}/plugin.go (83%) diff --git a/cmd/runner/runner.go b/cmd/runner/runner.go index 4d428a9b..df54eb00 100644 --- a/cmd/runner/runner.go +++ b/cmd/runner/runner.go @@ -53,7 +53,7 @@ import ( "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/plugins/modelselector/picker/random" "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/plugins/modelselector/picker/weightedrandom" inflightrequestsscorer "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/plugins/modelselector/scorer/inflightrequests" - queuettftscorer "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/plugins/modelselector/scorer/queuettft" + loadawarettftscorer "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/plugins/modelselector/scorer/loadawarettft" sessionaffinity "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/plugins/modelselector/scorer/sessionaffinity" "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/plugins/requesthandling/basemodelextractor" "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/plugins/requesthandling/bodyfieldtoheader" @@ -295,7 +295,7 @@ func (r *Runner) registerInTreePlugins() { plugin.Register(weightedrandom.WeightedRandomPickerType, weightedrandom.WeightedRandomPickerFactory) plugin.Register(modelselectorplugin.ModelSelectorPluginType, modelselectorplugin.ModelSelectorPluginFactory) plugin.Register(inflightrequestsscorer.PluginType, inflightrequestsscorer.ScorerFactory) - plugin.Register(queuettftscorer.PluginType, queuettftscorer.ScorerFactory) + plugin.Register(loadawarettftscorer.PluginType, loadawarettftscorer.ScorerFactory) plugin.Register(sessionaffinity.PluginType, sessionaffinity.ScorerFactory) } diff --git a/config/charts/payload-processor/templates/config.yaml b/config/charts/payload-processor/templates/config.yaml index 4785d58d..3ce225af 100644 --- a/config/charts/payload-processor/templates/config.yaml +++ b/config/charts/payload-processor/templates/config.yaml @@ -26,11 +26,4 @@ data: kind: PayloadProcessorConfig {{- .Values.payloadProcessor.customConfig | toYaml | nindent 4 }} {{- end }} - {{- if .Values.payloadProcessor.listModels }} - {{- $models := list }} - {{- range .Values.payloadProcessor.listModels }} - {{- $models = append $models (dict "name" .) }} - {{- end }} - models.json: {{ dict "models" $models | toJson | quote }} - {{- end }} --- diff --git a/examples/medianttft-values.yaml b/examples/medianttft-values.yaml index fbc92ab6..52039078 100644 --- a/examples/medianttft-values.yaml +++ b/examples/medianttft-values.yaml @@ -10,7 +10,7 @@ payloadProcessor: headerName: X-Gateway-Model-Name - type: base-model-to-header - type: model-selector - - type: queue-ttft-scorer + - type: load-aware-ttft-scorer # effectiveTTFT = P10Low + inflight x (P50 - P10Low) / inflightAtP50 # line through (0, P10Low) and (inflightAtP50, P50), extrapolated linearly parameters: @@ -32,7 +32,7 @@ payloadProcessor: plugins: request: - pluginRef: model-selector - - pluginRef: queue-ttft-scorer + - pluginRef: load-aware-ttft-scorer weight: 1.0 - pluginRef: max-score-picker - pluginRef: body-field-to-header diff --git a/pkg/framework/plugins/modelselector/scorer/queuettft/README.md b/pkg/framework/plugins/modelselector/scorer/loadawarettft/README.md similarity index 98% rename from pkg/framework/plugins/modelselector/scorer/queuettft/README.md rename to pkg/framework/plugins/modelselector/scorer/loadawarettft/README.md index 7941b855..4b3c66c2 100644 --- a/pkg/framework/plugins/modelselector/scorer/queuettft/README.md +++ b/pkg/framework/plugins/modelselector/scorer/loadawarettft/README.md @@ -1,4 +1,4 @@ -# Queue-TTFT Scorer +# Load-Aware TTFT Scorer Routes each request to the model with the lowest predicted TTFT under current load. @@ -55,7 +55,7 @@ will wait. No fitting, no tunable parameters — just two observed points. ## Parameters -### Scorer (`queue-ttft-scorer`) +### Scorer (`load-aware-ttft-scorer`) | Parameter | Default | Description | |---|---|---| diff --git a/pkg/framework/plugins/modelselector/scorer/queuettft/plugin.go b/pkg/framework/plugins/modelselector/scorer/loadawarettft/plugin.go similarity index 83% rename from pkg/framework/plugins/modelselector/scorer/queuettft/plugin.go rename to pkg/framework/plugins/modelselector/scorer/loadawarettft/plugin.go index 8cf557be..b176e3ef 100644 --- a/pkg/framework/plugins/modelselector/scorer/queuettft/plugin.go +++ b/pkg/framework/plugins/modelselector/scorer/loadawarettft/plugin.go @@ -14,12 +14,12 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Package queuettft scores models by predicted TTFT under current load. +// Package loadawarettft scores models by predicted TTFT under current load. // effectiveTTFT = P10Low + inflight × (P50 − P10Low) / inflightAtP50: // a line through (0, P10Low) and (inflightAtP50, P50), extrapolated linearly. // Under-observed models receive an optimistic seed (their own floor) so they // keep competing for traffic instead of stalling at a fixed 0.5 score. -package queuettft +package loadawarettft import ( "context" @@ -39,15 +39,15 @@ import ( ) const ( - PluginType = "queue-ttft-scorer" + PluginType = "load-aware-ttft-scorer" defaultExplorationRate = 0.0 // off by default; set e.g. 0.1 for 10% exploration ) -var _ modelselector.Scorer = &QueueTTFTScorer{} +var _ modelselector.Scorer = &LoadAwareTTFTScorer{} -// QueueTTFTScorerConfig holds optional parameters for the scorer plugin. -type QueueTTFTScorerConfig struct { +// LoadAwareTTFTScorerConfig holds optional parameters for the scorer plugin. +type LoadAwareTTFTScorerConfig struct { // ExplorationRate controls the probability of routing a request to an under-observed // model (UNOBSERVED or SEED state) instead of the trusted best model. A value of 0.1 // means ~10% of requests probe the under-observed model, preventing a burst of traffic @@ -56,13 +56,13 @@ type QueueTTFTScorerConfig struct { ExplorationRate float64 `json:"explorationRate,omitempty"` } -type QueueTTFTScorer struct { +type LoadAwareTTFTScorer struct { typedName plugin.TypedName explorationRate float64 } func ScorerFactory(name string, parameters json.RawMessage, _ plugin.Handle) (plugin.Plugin, error) { - cfg := QueueTTFTScorerConfig{ExplorationRate: defaultExplorationRate} + cfg := LoadAwareTTFTScorerConfig{ExplorationRate: defaultExplorationRate} if len(parameters) > 0 { if err := json.Unmarshal(parameters, &cfg); err != nil { return nil, fmt.Errorf("failed to parse parameters for plugin %q: %w", name, err) @@ -71,22 +71,22 @@ func ScorerFactory(name string, parameters json.RawMessage, _ plugin.Handle) (pl if cfg.ExplorationRate < 0 || cfg.ExplorationRate > 1 { return nil, fmt.Errorf("explorationRate must be in [0, 1] for plugin %q", name) } - return NewQueueTTFTScorer().WithName(name).WithExplorationRate(cfg.ExplorationRate), nil + return NewLoadAwareTTFTScorer().WithName(name).WithExplorationRate(cfg.ExplorationRate), nil } -func NewQueueTTFTScorer() *QueueTTFTScorer { - return &QueueTTFTScorer{ +func NewLoadAwareTTFTScorer() *LoadAwareTTFTScorer { + return &LoadAwareTTFTScorer{ typedName: plugin.TypedName{Type: PluginType, Name: PluginType}, explorationRate: defaultExplorationRate, } } -func (s *QueueTTFTScorer) TypedName() plugin.TypedName { return s.typedName } -func (s *QueueTTFTScorer) WithName(name string) *QueueTTFTScorer { +func (s *LoadAwareTTFTScorer) TypedName() plugin.TypedName { return s.typedName } +func (s *LoadAwareTTFTScorer) WithName(name string) *LoadAwareTTFTScorer { s.typedName.Name = name return s } -func (s *QueueTTFTScorer) WithExplorationRate(r float64) *QueueTTFTScorer { +func (s *LoadAwareTTFTScorer) WithExplorationRate(r float64) *LoadAwareTTFTScorer { s.explorationRate = r return s } @@ -104,7 +104,7 @@ type modelEval struct { // Cold models (no floor) are seeded at the best observed TTFT; if every model is cold, all // score 1.0. With explorationRate > 0, under-observed models (not yet calibrated) are // suppressed to 0 with probability (1 - explorationRate) to throttle probing traffic. -func (s *QueueTTFTScorer) Score(ctx context.Context, cycleState *plugin.CycleState, _ *requesthandling.InferenceRequest, models []datalayer.Model) map[datalayer.Model]float64 { +func (s *LoadAwareTTFTScorer) Score(ctx context.Context, cycleState *plugin.CycleState, _ *requesthandling.InferenceRequest, models []datalayer.Model) map[datalayer.Model]float64 { evals := make(map[datalayer.Model]*modelEval, len(models)) minEff := math.MaxFloat64 anyObserved := false @@ -159,7 +159,7 @@ func (s *QueueTTFTScorer) Score(ctx context.Context, cycleState *plugin.CycleSta if dl := log.FromContext(ctx).V(logutil.DEBUG); dl.Enabled() { for _, model := range models { e := evals[model] - dl.Info("queue-ttft score", "model", model.GetName(), + dl.Info("load-aware-ttft score", "model", model.GetName(), "effectiveTTFT", e.eff, "score", scores[model], "trusted", e.trusted) } } From fbc1739b334d8604747b6b4627592de7279df6da Mon Sep 17 00:00:00 2001 From: Mohammad Date: Thu, 2 Jul 2026 14:00:19 +0300 Subject: [PATCH 20/27] Move example values to readme. Signed-off-by: Mohammad --- examples/medianttft-values.yaml | 44 ---------------- .../scorer/loadawarettft/README.md | 52 +++++++++++++++++++ 2 files changed, 52 insertions(+), 44 deletions(-) delete mode 100644 examples/medianttft-values.yaml diff --git a/examples/medianttft-values.yaml b/examples/medianttft-values.yaml deleted file mode 100644 index 52039078..00000000 --- a/examples/medianttft-values.yaml +++ /dev/null @@ -1,44 +0,0 @@ -payloadProcessor: - listModels: - - Qwen/Qwen3-8B - - Qwen/Qwen3-32B - customConfig: - plugins: - - type: body-field-to-header - parameters: - fieldName: model - headerName: X-Gateway-Model-Name - - type: base-model-to-header - - type: model-selector - - type: load-aware-ttft-scorer - # effectiveTTFT = P10Low + inflight x (P50 - P10Low) / inflightAtP50 - # line through (0, P10Low) and (inflightAtP50, P50), extrapolated linearly - parameters: - explorationRate: 0.1 # 10% of requests probe under-observed models; 0 = disabled - - type: max-score-picker - - type: ttft-percentile-extractor - parameters: - intervalDuration: 1s - windowSize: 5000 - maxObservationAge: 3m - lowLoadWindowAge: 1h - maxRequests: 100 - minRequests: 20 - - type: model-config-datasource - parameters: - modelsPath: /config/models.json - profiles: - - name: default - plugins: - request: - - pluginRef: model-selector - - pluginRef: load-aware-ttft-scorer - weight: 1.0 - - pluginRef: max-score-picker - - pluginRef: body-field-to-header - - pluginRef: base-model-to-header - datalayer: - extractors: - - pluginRef: ttft-percentile-extractor - datasources: - - pluginRef: model-config-datasource diff --git a/pkg/framework/plugins/modelselector/scorer/loadawarettft/README.md b/pkg/framework/plugins/modelselector/scorer/loadawarettft/README.md index 4b3c66c2..80607032 100644 --- a/pkg/framework/plugins/modelselector/scorer/loadawarettft/README.md +++ b/pkg/framework/plugins/modelselector/scorer/loadawarettft/README.md @@ -72,6 +72,58 @@ will wait. No fitting, no tunable parameters — just two observed points. | `floorInterval` | 1m | How often P10Low is recomputed (slow-moving floor; cheaper than every interval) | | `windowSize` | 5000 | Ring buffer capacity (~200 KB per model) | +## Example configuration + +An end-to-end Helm values override wiring the scorer together with the TTFT extractor, +the model-config datasource, and a picker: + +```yaml +payloadProcessor: + customConfig: + plugins: + - type: body-field-to-header + parameters: + fieldName: model + headerName: X-Gateway-Model-Name + - type: base-model-to-header + - type: model-selector + - type: load-aware-ttft-scorer + # effectiveTTFT = P10Low + inflight x (P50 - P10Low) / inflightAtP50 + # line through (0, P10Low) and (inflightAtP50, P50), extrapolated linearly + parameters: + explorationRate: 0.1 # 10% of requests probe under-observed models; 0 = disabled + - type: max-score-picker + - type: ttft-percentile-extractor + parameters: + intervalDuration: 1s + windowSize: 5000 + maxObservationAge: 3m + lowLoadWindowAge: 1h + maxRequests: 100 + minRequests: 20 + - type: model-config-datasource + parameters: + modelsPath: /config/models.json + profiles: + - name: default + plugins: + request: + - pluginRef: model-selector + - pluginRef: load-aware-ttft-scorer + weight: 1.0 + - pluginRef: max-score-picker + - pluginRef: body-field-to-header + - pluginRef: base-model-to-header + datalayer: + extractors: + - pluginRef: ttft-percentile-extractor + datasources: + - pluginRef: model-config-datasource +``` + +The scorer requires the `ttft-percentile-extractor` in `datalayer.extractors`, and a model +list (here via `model-config-datasource`) so model selection has candidates. + ## Possible Enhancements ### Score-proportional picker From 45c3a1eee42e2bd38b8c24027dd9005d6860bd4e Mon Sep 17 00:00:00 2001 From: Mohammad Date: Thu, 2 Jul 2026 15:07:04 +0300 Subject: [PATCH 21/27] Rename the scorer to ttft-aware-scorer. Signed-off-by: Mohammad --- cmd/runner/runner.go | 4 +-- .../{loadawarettft => ttftaware}/README.md | 8 ++--- .../{loadawarettft => ttftaware}/plugin.go | 32 +++++++++---------- 3 files changed, 22 insertions(+), 22 deletions(-) rename pkg/framework/plugins/modelselector/scorer/{loadawarettft => ttftaware}/README.md (97%) rename pkg/framework/plugins/modelselector/scorer/{loadawarettft => ttftaware}/plugin.go (83%) diff --git a/cmd/runner/runner.go b/cmd/runner/runner.go index df54eb00..0401bbb4 100644 --- a/cmd/runner/runner.go +++ b/cmd/runner/runner.go @@ -53,8 +53,8 @@ import ( "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/plugins/modelselector/picker/random" "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/plugins/modelselector/picker/weightedrandom" inflightrequestsscorer "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/plugins/modelselector/scorer/inflightrequests" - loadawarettftscorer "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/plugins/modelselector/scorer/loadawarettft" sessionaffinity "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/plugins/modelselector/scorer/sessionaffinity" + ttftawarescorer "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/plugins/modelselector/scorer/ttftaware" "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/plugins/requesthandling/basemodelextractor" "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/plugins/requesthandling/bodyfieldtoheader" modelselectorplugin "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/plugins/requesthandling/modelselector" @@ -295,7 +295,7 @@ func (r *Runner) registerInTreePlugins() { plugin.Register(weightedrandom.WeightedRandomPickerType, weightedrandom.WeightedRandomPickerFactory) plugin.Register(modelselectorplugin.ModelSelectorPluginType, modelselectorplugin.ModelSelectorPluginFactory) plugin.Register(inflightrequestsscorer.PluginType, inflightrequestsscorer.ScorerFactory) - plugin.Register(loadawarettftscorer.PluginType, loadawarettftscorer.ScorerFactory) + plugin.Register(ttftawarescorer.PluginType, ttftawarescorer.ScorerFactory) plugin.Register(sessionaffinity.PluginType, sessionaffinity.ScorerFactory) } diff --git a/pkg/framework/plugins/modelselector/scorer/loadawarettft/README.md b/pkg/framework/plugins/modelselector/scorer/ttftaware/README.md similarity index 97% rename from pkg/framework/plugins/modelselector/scorer/loadawarettft/README.md rename to pkg/framework/plugins/modelselector/scorer/ttftaware/README.md index 80607032..7989f85d 100644 --- a/pkg/framework/plugins/modelselector/scorer/loadawarettft/README.md +++ b/pkg/framework/plugins/modelselector/scorer/ttftaware/README.md @@ -1,4 +1,4 @@ -# Load-Aware TTFT Scorer +# TTFT-Aware Scorer Routes each request to the model with the lowest predicted TTFT under current load. @@ -55,7 +55,7 @@ will wait. No fitting, no tunable parameters — just two observed points. ## Parameters -### Scorer (`load-aware-ttft-scorer`) +### Scorer (`ttft-aware-scorer`) | Parameter | Default | Description | |---|---|---| @@ -87,7 +87,7 @@ payloadProcessor: headerName: X-Gateway-Model-Name - type: base-model-to-header - type: model-selector - - type: load-aware-ttft-scorer + - type: ttft-aware-scorer # effectiveTTFT = P10Low + inflight x (P50 - P10Low) / inflightAtP50 # line through (0, P10Low) and (inflightAtP50, P50), extrapolated linearly parameters: @@ -109,7 +109,7 @@ payloadProcessor: plugins: request: - pluginRef: model-selector - - pluginRef: load-aware-ttft-scorer + - pluginRef: ttft-aware-scorer weight: 1.0 - pluginRef: max-score-picker - pluginRef: body-field-to-header diff --git a/pkg/framework/plugins/modelselector/scorer/loadawarettft/plugin.go b/pkg/framework/plugins/modelselector/scorer/ttftaware/plugin.go similarity index 83% rename from pkg/framework/plugins/modelselector/scorer/loadawarettft/plugin.go rename to pkg/framework/plugins/modelselector/scorer/ttftaware/plugin.go index b176e3ef..54e0020a 100644 --- a/pkg/framework/plugins/modelselector/scorer/loadawarettft/plugin.go +++ b/pkg/framework/plugins/modelselector/scorer/ttftaware/plugin.go @@ -14,12 +14,12 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Package loadawarettft scores models by predicted TTFT under current load. +// Package ttftaware scores models by predicted TTFT under current load. // effectiveTTFT = P10Low + inflight × (P50 − P10Low) / inflightAtP50: // a line through (0, P10Low) and (inflightAtP50, P50), extrapolated linearly. // Under-observed models receive an optimistic seed (their own floor) so they // keep competing for traffic instead of stalling at a fixed 0.5 score. -package loadawarettft +package ttftaware import ( "context" @@ -39,15 +39,15 @@ import ( ) const ( - PluginType = "load-aware-ttft-scorer" + PluginType = "ttft-aware-scorer" defaultExplorationRate = 0.0 // off by default; set e.g. 0.1 for 10% exploration ) -var _ modelselector.Scorer = &LoadAwareTTFTScorer{} +var _ modelselector.Scorer = &TTFTAwareScorer{} -// LoadAwareTTFTScorerConfig holds optional parameters for the scorer plugin. -type LoadAwareTTFTScorerConfig struct { +// TTFTAwareScorerConfig holds optional parameters for the scorer plugin. +type TTFTAwareScorerConfig struct { // ExplorationRate controls the probability of routing a request to an under-observed // model (UNOBSERVED or SEED state) instead of the trusted best model. A value of 0.1 // means ~10% of requests probe the under-observed model, preventing a burst of traffic @@ -56,13 +56,13 @@ type LoadAwareTTFTScorerConfig struct { ExplorationRate float64 `json:"explorationRate,omitempty"` } -type LoadAwareTTFTScorer struct { +type TTFTAwareScorer struct { typedName plugin.TypedName explorationRate float64 } func ScorerFactory(name string, parameters json.RawMessage, _ plugin.Handle) (plugin.Plugin, error) { - cfg := LoadAwareTTFTScorerConfig{ExplorationRate: defaultExplorationRate} + cfg := TTFTAwareScorerConfig{ExplorationRate: defaultExplorationRate} if len(parameters) > 0 { if err := json.Unmarshal(parameters, &cfg); err != nil { return nil, fmt.Errorf("failed to parse parameters for plugin %q: %w", name, err) @@ -71,22 +71,22 @@ func ScorerFactory(name string, parameters json.RawMessage, _ plugin.Handle) (pl if cfg.ExplorationRate < 0 || cfg.ExplorationRate > 1 { return nil, fmt.Errorf("explorationRate must be in [0, 1] for plugin %q", name) } - return NewLoadAwareTTFTScorer().WithName(name).WithExplorationRate(cfg.ExplorationRate), nil + return NewTTFTAwareScorer().WithName(name).WithExplorationRate(cfg.ExplorationRate), nil } -func NewLoadAwareTTFTScorer() *LoadAwareTTFTScorer { - return &LoadAwareTTFTScorer{ +func NewTTFTAwareScorer() *TTFTAwareScorer { + return &TTFTAwareScorer{ typedName: plugin.TypedName{Type: PluginType, Name: PluginType}, explorationRate: defaultExplorationRate, } } -func (s *LoadAwareTTFTScorer) TypedName() plugin.TypedName { return s.typedName } -func (s *LoadAwareTTFTScorer) WithName(name string) *LoadAwareTTFTScorer { +func (s *TTFTAwareScorer) TypedName() plugin.TypedName { return s.typedName } +func (s *TTFTAwareScorer) WithName(name string) *TTFTAwareScorer { s.typedName.Name = name return s } -func (s *LoadAwareTTFTScorer) WithExplorationRate(r float64) *LoadAwareTTFTScorer { +func (s *TTFTAwareScorer) WithExplorationRate(r float64) *TTFTAwareScorer { s.explorationRate = r return s } @@ -104,7 +104,7 @@ type modelEval struct { // Cold models (no floor) are seeded at the best observed TTFT; if every model is cold, all // score 1.0. With explorationRate > 0, under-observed models (not yet calibrated) are // suppressed to 0 with probability (1 - explorationRate) to throttle probing traffic. -func (s *LoadAwareTTFTScorer) Score(ctx context.Context, cycleState *plugin.CycleState, _ *requesthandling.InferenceRequest, models []datalayer.Model) map[datalayer.Model]float64 { +func (s *TTFTAwareScorer) Score(ctx context.Context, cycleState *plugin.CycleState, _ *requesthandling.InferenceRequest, models []datalayer.Model) map[datalayer.Model]float64 { evals := make(map[datalayer.Model]*modelEval, len(models)) minEff := math.MaxFloat64 anyObserved := false @@ -159,7 +159,7 @@ func (s *LoadAwareTTFTScorer) Score(ctx context.Context, cycleState *plugin.Cycl if dl := log.FromContext(ctx).V(logutil.DEBUG); dl.Enabled() { for _, model := range models { e := evals[model] - dl.Info("load-aware-ttft score", "model", model.GetName(), + dl.Info("ttft-aware score", "model", model.GetName(), "effectiveTTFT", e.eff, "score", scores[model], "trusted", e.trusted) } } From 7b60367416a3907ef467f46cebfd076039f1892d Mon Sep 17 00:00:00 2001 From: Mohammad Date: Mon, 13 Jul 2026 14:32:47 +0300 Subject: [PATCH 22/27] Fix review comments. Signed-off-by: Mohammad --- .../plugins/modelselector/scorer/ttftaware/plugin.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pkg/framework/plugins/modelselector/scorer/ttftaware/plugin.go b/pkg/framework/plugins/modelselector/scorer/ttftaware/plugin.go index 54e0020a..03dd7cfd 100644 --- a/pkg/framework/plugins/modelselector/scorer/ttftaware/plugin.go +++ b/pkg/framework/plugins/modelselector/scorer/ttftaware/plugin.go @@ -82,10 +82,12 @@ func NewTTFTAwareScorer() *TTFTAwareScorer { } func (s *TTFTAwareScorer) TypedName() plugin.TypedName { return s.typedName } + func (s *TTFTAwareScorer) WithName(name string) *TTFTAwareScorer { s.typedName.Name = name return s } + func (s *TTFTAwareScorer) WithExplorationRate(r float64) *TTFTAwareScorer { s.explorationRate = r return s @@ -170,10 +172,10 @@ func (s *TTFTAwareScorer) Score(ctx context.Context, cycleState *plugin.CycleSta // metricsFor reads the TTFT percentile metrics an extractor published for the model. // A missing or malformed attribute yields the zero value, which reads as truly cold. func metricsFor(model datalayer.Model) ttftpercentile.TTFTPercentileMetrics { - if val, ok := model.GetAttributes().Get(ttftpercentile.AttributeKey); ok { - if m, ok := val.(ttftpercentile.TTFTPercentileMetrics); ok { - return m - } + if val, err := datalayer.ReadAttributeKey[ttftpercentile.TTFTPercentileMetrics]( + model.GetAttributes(), ttftpercentile.AttributeKey, + ); err == nil { + return val } return ttftpercentile.TTFTPercentileMetrics{} } From d2dd9af72dc92cbc8a4d9787542c453e5102eb31 Mon Sep 17 00:00:00 2001 From: Mohammad Date: Sun, 19 Jul 2026 10:12:08 +0300 Subject: [PATCH 23/27] Update scorer. Signed-off-by: Mohammad --- .../modelselector/scorer/ttftaware/README.md | 85 +++++++++------ .../modelselector/scorer/ttftaware/plugin.go | 103 +++++++++++++----- 2 files changed, 125 insertions(+), 63 deletions(-) diff --git a/pkg/framework/plugins/modelselector/scorer/ttftaware/README.md b/pkg/framework/plugins/modelselector/scorer/ttftaware/README.md index 7989f85d..44a6cfa9 100644 --- a/pkg/framework/plugins/modelselector/scorer/ttftaware/README.md +++ b/pkg/framework/plugins/modelselector/scorer/ttftaware/README.md @@ -6,52 +6,66 @@ Routes each request to the model with the lowest predicted TTFT under current lo Every TTFT decomposes as `TTFT = prefill_time + queue_wait`. -**P10Low** — hardware-bound service floor: +**P10Low** — hardware-bound service floor (published by the extractor): -Computed from a long window (default 1h) using all observations regardless of inflight level: -1. Find the P10 TTFT threshold across all observations in the window -2. Take the P10 of only the observations at or below that threshold (~P1 of all) +The extractor keeps a bounded history of per-bucket P10s: once per `bucketDuration` it records +the P10 TTFT of that bucket, and `P10Low` is the P10 of that history. When the history is full +(`bucketHistorySize` entries) the smallest and largest entries are evicted — not the oldest — so +a single anomalously fast or slow bucket never sticks. -This isolates the fastest requests in the window — those with the least queue wait — without -requiring the model to have idle periods. P10Low is hardware-bound and stable: prefill time -does not change with queue depth, concurrency level, or scale events. +This is load-invariant and robust: the history spans idle and busy buckets, and taking a low +percentile of it locks onto the idle buckets (the true prefill floor) instead of drifting up with +recent load. Prefill time does not change with queue depth or concurrency, so the floor is stable. -**P50 and inflightAtP50** — current operating point (short window, default 3m / 100 requests): +**Operating points** — measured over a short window (default 3m / 100 requests, kept responsive): ``` -P50 = 50th percentile TTFT -inflightAtP50 = average inflight_at_dispatch of observations in the P40-P60 band +P25, P50 = 25th / 50th percentile TTFT +inflightAtP25 = average inflight_at_dispatch in the P15-P35 band +inflightAtP50 = average inflight_at_dispatch in the P40-P60 band ``` -The short window keeps P50 responsive to current load. Averaging over a band rather than -a single observation makes inflightAtP50 more stable. +Averaging inflight over a band rather than a single observation stabilises the estimate. -**effectiveTTFT** — predicted TTFT for a request arriving now: +**effectiveTTFT** — predicted TTFT for a request arriving now. + +A line through the high operating point `(inflightAtP50, P50)` and a low anchor that blends between +the in-cloud point `(inflightAtP25, P25)` and the load-free floor `(0, P10Low)`: ``` -effectiveTTFT = P10Low + inflight x (P50 - P10Low) / inflightAtP50 +w = clamp((inflightAtP50 - inflightAtP25) / anchorGapScale, 0, 1) +lowInflight = w * inflightAtP25 +lowTTFT = w * P25 + (1 - w) * P10Low +effectiveTTFT = lowTTFT + (inflight - lowInflight) * (P50 - lowTTFT) / (inflightAtP50 - lowInflight) ``` -Falls back to P10Low when P50 is not yet available or equals the floor. +Clamped to `>= P10Low`. Under-observed models (not yet calibrated) seed at `P10Low`. **Score:** ``` score = (maxTTFT - effectiveTTFT) / (maxTTFT - minTTFT) ``` -Under-observed models (UNOBSERVED or SEED state) receive an optimistic high score. -With `explorationRate > 0`, that high score is suppressed to 0 with probability -`(1 - explorationRate)` so only ~`explorationRate` fraction of requests are routed -to the under-observed model for calibration probing. +Under-observed models (UNOBSERVED or SEED state) receive an optimistic high score. With +`explorationRate > 0`, that high score is suppressed to 0 with probability `(1 - explorationRate)` +so only ~`explorationRate` of requests probe the under-observed model for calibration. ## Why it works physically -When more requests are in flight, a new request has to wait longer in the queue before -the model processes it. The longer the queue, the higher the TTFT. This wait grows -roughly in proportion to the number of in-flight requests. +When more requests are in flight, a new request waits longer in the queue, so TTFT rises with +inflight — and it rises *faster than linearly* as the server approaches saturation (queueing). + +The scorer draws a line through two points it has actually observed: -The scorer draws a straight line through two points it has actually observed: +- fast requests ran at lower load: `(inflightAtP25, P25)` +- median requests at higher load: `(inflightAtP50, P50)` -- when there is no queue (`inflight = 0`): TTFT = P10Low (just the raw prefill time) -- at the recent median load (`inflight = inflightAtP50`): TTFT = P50 +Because both anchors sit *inside* the observed load cloud, the line follows the **local slope** of +that convex curve — so it does not systematically under-predict the way a single chord drawn from a +synthetic zero-load floor does. -It then reads off that line at the current inflight to predict what the next request -will wait. No fitting, no tunable parameters — just two observed points. +At low load the two anchors collapse together (every request sees similar, low inflight), so their +slope is ill-defined. The blend weight `w` handles this smoothly: as the inflight gap shrinks, `w → +0` slides the low anchor down to `(0, P10Low)`, recovering the stable floor chord. There is no +threshold and no discontinuity — the prediction transitions continuously between the two regimes, +and the denominator `inflightAtP50 - w*inflightAtP25` can never collapse. The only knob, +`anchorGapScale`, is a numerical-conditioning scale (how much inflight separation counts as "well +separated"), not a fitted parameter. ## Parameters @@ -59,18 +73,19 @@ will wait. No fitting, no tunable parameters — just two observed points. | Parameter | Default | Description | |---|---|---| -| `explorationRate` | 0.0 | Fraction of requests routed to under-observed models for calibration probing. 0 = all requests go to the trusted winner; 0.1 = ~10% probe the under-observed model. | +| `explorationRate` | 0.0 | Fraction of requests routed to under-observed models for calibration probing. 0 = all traffic to the trusted winner; 0.1 = ~10% probe. | +| `anchorGapScale` | 2.0 | Inflight separation (`inflightAtP50 - inflightAtP25`) at which the prediction fully trusts the in-cloud secant; below it the low anchor blends toward the floor chord. Must be > 0. | ### Extractor (`ttft-percentile-extractor`) | Parameter | Default | Description | |---|---|---| -| `maxObservationAge` | 3m | Time bound for the short window (P50 / P10) | +| `maxObservationAge` | 3m | Time bound for the short window (P25 / P50 / P10) | | `maxRequests` | 100 | Cap the short window to the most recent N observations | -| `minRequests` | 10 | Minimum capped-window count before the scorer trusts the formula | -| `lowLoadWindowAge` | 1h | Window for two-level P10Low (long = stable hardware floor) | -| `floorInterval` | 1m | How often P10Low is recomputed (slow-moving floor; cheaper than every interval) | +| `minRequests` | 10 | Minimum capped-window count before the scorer trusts the operating point | | `windowSize` | 5000 | Ring buffer capacity (~200 KB per model) | +| `bucketDuration` | 1m | Window for each floor-history entry's P10; keep `<= maxObservationAge` | +| `bucketHistorySize` | 720 | Per-bucket P10s kept for the floor (`bucketDuration * bucketHistorySize` = horizon, 12h); min/max evicted when full | ## Example configuration @@ -88,19 +103,19 @@ payloadProcessor: - type: base-model-to-header - type: model-selector - type: ttft-aware-scorer - # effectiveTTFT = P10Low + inflight x (P50 - P10Low) / inflightAtP50 - # line through (0, P10Low) and (inflightAtP50, P50), extrapolated linearly parameters: explorationRate: 0.1 # 10% of requests probe under-observed models; 0 = disabled + anchorGapScale: 2.0 # inflight separation at which the in-cloud secant is fully trusted - type: max-score-picker - type: ttft-percentile-extractor parameters: intervalDuration: 1s windowSize: 5000 maxObservationAge: 3m - lowLoadWindowAge: 1h maxRequests: 100 minRequests: 20 + bucketDuration: 1m + bucketHistorySize: 720 - type: model-config-datasource parameters: modelsPath: /config/models.json diff --git a/pkg/framework/plugins/modelselector/scorer/ttftaware/plugin.go b/pkg/framework/plugins/modelselector/scorer/ttftaware/plugin.go index 03dd7cfd..571623b9 100644 --- a/pkg/framework/plugins/modelselector/scorer/ttftaware/plugin.go +++ b/pkg/framework/plugins/modelselector/scorer/ttftaware/plugin.go @@ -14,11 +14,10 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Package ttftaware scores models by predicted TTFT under current load. -// effectiveTTFT = P10Low + inflight × (P50 − P10Low) / inflightAtP50: -// a line through (0, P10Low) and (inflightAtP50, P50), extrapolated linearly. -// Under-observed models receive an optimistic seed (their own floor) so they -// keep competing for traffic instead of stalling at a fixed 0.5 score. +// Package ttftaware routes each request to the model with the lowest predicted TTFT under +// current load. Prediction is a line through (inflightAtP50, P50) and a low anchor blended +// between the in-cloud point (inflightAtP25, P25) and the floor (0, P10Low). See README.md +// for the equations and rationale. package ttftaware import ( @@ -42,6 +41,7 @@ const ( PluginType = "ttft-aware-scorer" defaultExplorationRate = 0.0 // off by default; set e.g. 0.1 for 10% exploration + defaultAnchorGapScale = 2.0 // inflight separation at which the blend fully trusts the secant ) var _ modelselector.Scorer = &TTFTAwareScorer{} @@ -54,15 +54,22 @@ type TTFTAwareScorerConfig struct { // before the first responses return and P50 is calibrated. // Range [0, 1]. Default 0 (disabled — every request goes to the winner). ExplorationRate float64 `json:"explorationRate,omitempty"` + // AnchorGapScale is the inflight separation at which the prediction fully trusts the in-cloud + // secant; below it the low anchor blends toward the floor chord. Must be > 0. Default 2. + AnchorGapScale float64 `json:"anchorGapScale,omitempty"` } type TTFTAwareScorer struct { typedName plugin.TypedName explorationRate float64 + anchorGapScale float64 } func ScorerFactory(name string, parameters json.RawMessage, _ plugin.Handle) (plugin.Plugin, error) { - cfg := TTFTAwareScorerConfig{ExplorationRate: defaultExplorationRate} + cfg := TTFTAwareScorerConfig{ + ExplorationRate: defaultExplorationRate, + AnchorGapScale: defaultAnchorGapScale, + } if len(parameters) > 0 { if err := json.Unmarshal(parameters, &cfg); err != nil { return nil, fmt.Errorf("failed to parse parameters for plugin %q: %w", name, err) @@ -71,13 +78,20 @@ func ScorerFactory(name string, parameters json.RawMessage, _ plugin.Handle) (pl if cfg.ExplorationRate < 0 || cfg.ExplorationRate > 1 { return nil, fmt.Errorf("explorationRate must be in [0, 1] for plugin %q", name) } - return NewTTFTAwareScorer().WithName(name).WithExplorationRate(cfg.ExplorationRate), nil + if cfg.AnchorGapScale <= 0 { + return nil, fmt.Errorf("anchorGapScale must be > 0 for plugin %q", name) + } + return NewTTFTAwareScorer(). + WithName(name). + WithExplorationRate(cfg.ExplorationRate). + WithAnchorGapScale(cfg.AnchorGapScale), nil } func NewTTFTAwareScorer() *TTFTAwareScorer { return &TTFTAwareScorer{ typedName: plugin.TypedName{Type: PluginType, Name: PluginType}, explorationRate: defaultExplorationRate, + anchorGapScale: defaultAnchorGapScale, } } @@ -93,6 +107,11 @@ func (s *TTFTAwareScorer) WithExplorationRate(r float64) *TTFTAwareScorer { return s } +func (s *TTFTAwareScorer) WithAnchorGapScale(g float64) *TTFTAwareScorer { + s.anchorGapScale = g + return s +} + // modelEval is the scorer's per-model working state for one Score call. type modelEval struct { metrics ttftpercentile.TTFTPercentileMetrics @@ -104,23 +123,28 @@ type modelEval struct { // Score ranks models by predicted TTFT: score = (maxTTFT − effectiveTTFT) / (maxTTFT − minTTFT). // // Cold models (no floor) are seeded at the best observed TTFT; if every model is cold, all -// score 1.0. With explorationRate > 0, under-observed models (not yet calibrated) are -// suppressed to 0 with probability (1 - explorationRate) to throttle probing traffic. +// score 1.0. With explorationRate > 0, each request makes one exploration decision: with +// probability explorationRate the under-observed models keep their scores (so one may win and +// get a calibration probe), otherwise they are suppressed to 0 — but only when a calibrated +// model exists to take the traffic. func (s *TTFTAwareScorer) Score(ctx context.Context, cycleState *plugin.CycleState, _ *requesthandling.InferenceRequest, models []datalayer.Model) map[datalayer.Model]float64 { evals := make(map[datalayer.Model]*modelEval, len(models)) minEff := math.MaxFloat64 + maxEff := 0.0 anyObserved := false + anyTrusted := false for _, model := range models { m := metricsFor(model) - eff, trusted := m.Predict() - e := &modelEval{metrics: m, eff: eff, trusted: trusted, observed: m.Floor() > 0} - evals[model] = e - if e.observed { + eff, trusted, observed := s.predict(m) + evals[model] = &modelEval{metrics: m, eff: eff, trusted: trusted, observed: observed} + if observed { anyObserved = true - if eff < minEff { - minEff = eff - } + minEff = min(minEff, eff) + maxEff = max(maxEff, eff) // cold models seed to minEff, so they can never be the max + } + if trusted { + anyTrusted = true } } @@ -134,26 +158,23 @@ func (s *TTFTAwareScorer) Score(ctx context.Context, cycleState *plugin.CycleSta return scores } - // Seed cold models at the best observed TTFT (optimistic). Seeds equal minEff, so the - // range spans the observed models and maxEff is their slowest. - maxEff := 0.0 - for _, e := range evals { - if !e.observed { - e.eff = minEff - } - if e.eff > maxEff { - maxEff = e.eff - } - } + // One exploration decision per request: with probability (1 - explorationRate) suppress all + // under-observed models so only calibrated models compete; otherwise leave them scored so an + // under-observed model can win and get a calibration probe. Only suppress when a calibrated + // model exists to take the traffic. + suppressUntrusted := s.explorationRate > 0 && anyTrusted && rand.Float64() >= s.explorationRate for _, model := range models { e := evals[model] + if !e.observed { + e.eff = minEff // seed cold models at the best observed TTFT (optimistic) + } if maxEff == minEff { scores[model] = 1.0 } else { scores[model] = (maxEff - e.eff) / (maxEff - minEff) } - if s.explorationRate > 0 && !e.trusted && rand.Float64() >= s.explorationRate { + if suppressUntrusted && !e.trusted { scores[model] = 0 } } @@ -169,6 +190,32 @@ func (s *TTFTAwareScorer) Score(ctx context.Context, cycleState *plugin.CycleSta return scores } +// predict returns the model's effective TTFT and whether its operating point is trusted +// (calibrated) and observed (has a floor). Cold models return (0, false, false); observed but +// uncalibrated models seed at the floor. See README.md for the blended-secant equation. +func (s *TTFTAwareScorer) predict(m ttftpercentile.TTFTPercentileMetrics) (eff float64, trusted, observed bool) { + floor := m.Floor() + if floor == 0 { + return 0, false, false + } + if !(m.RecentN >= m.MinRequests && m.InflightAtP50 > 0 && m.P50TTFT > floor) { + return floor, false, true // optimistic seed at the floor + } + + // Blend the low anchor between (iP25, P25) and (0, floor) by the anchor separation; w→1 + // under load gives the in-cloud secant, w→0 at low load gives the floor chord. + w := max(0, min(1, (m.InflightAtP50-m.InflightAtP25)/s.anchorGapScale)) + lowInflight := w * m.InflightAtP25 + lowTTFT := w*m.P25TTFT + (1-w)*floor + + cur := float64(m.Requests) + eff = lowTTFT + (cur-lowInflight)*(m.P50TTFT-lowTTFT)/(m.InflightAtP50-lowInflight) + if eff < floor { + eff = floor + } + return eff, true, true +} + // metricsFor reads the TTFT percentile metrics an extractor published for the model. // A missing or malformed attribute yields the zero value, which reads as truly cold. func metricsFor(model datalayer.Model) ttftpercentile.TTFTPercentileMetrics { From 188d7f9311818b84db1193e1583447957d2a11f2 Mon Sep 17 00:00:00 2001 From: Mohammad Date: Wed, 22 Jul 2026 16:01:26 +0300 Subject: [PATCH 24/27] Update exploration method. Signed-off-by: Mohammad --- .../modelselector/scorer/ttftaware/README.md | 145 +++++++++--------- .../modelselector/scorer/ttftaware/plugin.go | 30 ++-- 2 files changed, 87 insertions(+), 88 deletions(-) diff --git a/pkg/framework/plugins/modelselector/scorer/ttftaware/README.md b/pkg/framework/plugins/modelselector/scorer/ttftaware/README.md index 44a6cfa9..53838d21 100644 --- a/pkg/framework/plugins/modelselector/scorer/ttftaware/README.md +++ b/pkg/framework/plugins/modelselector/scorer/ttftaware/README.md @@ -2,50 +2,72 @@ Routes each request to the model with the lowest predicted TTFT under current load. -## Equations +It consumes the per-model snapshot published by the +[TTFT percentile extractor](../../../datalayer/ttftpercentile/README.md) — the service floor +`P10Low`, the operating points `P25` / `P50`, and their banded inflight averages `inflightAtP25` / +`inflightAtP50` — and turns them into a prediction and a score. See the extractor README for how +those inputs are measured. -Every TTFT decomposes as `TTFT = prefill_time + queue_wait`. +## Prediction — `effectiveTTFT` -**P10Low** — hardware-bound service floor (published by the extractor): +The predicted TTFT for a request arriving now is a line through the high operating point +`(inflightAtP50, P50)` and a low anchor that blends between the in-cloud point +`(inflightAtP25, P25)` and the load-free floor `(0, P10Low)`: -The extractor keeps a bounded history of per-bucket P10s: once per `bucketDuration` it records -the P10 TTFT of that bucket, and `P10Low` is the P10 of that history. When the history is full -(`bucketHistorySize` entries) the smallest and largest entries are evicted — not the oldest — so -a single anomalously fast or slow bucket never sticks. - -This is load-invariant and robust: the history spans idle and busy buckets, and taking a low -percentile of it locks onto the idle buckets (the true prefill floor) instead of drifting up with -recent load. Prefill time does not change with queue depth or concurrency, so the floor is stable. - -**Operating points** — measured over a short window (default 3m / 100 requests, kept responsive): -``` -P25, P50 = 25th / 50th percentile TTFT -inflightAtP25 = average inflight_at_dispatch in the P15-P35 band -inflightAtP50 = average inflight_at_dispatch in the P40-P60 band -``` -Averaging inflight over a band rather than a single observation stabilises the estimate. - -**effectiveTTFT** — predicted TTFT for a request arriving now. - -A line through the high operating point `(inflightAtP50, P50)` and a low anchor that blends between -the in-cloud point `(inflightAtP25, P25)` and the load-free floor `(0, P10Low)`: ``` w = clamp((inflightAtP50 - inflightAtP25) / anchorGapScale, 0, 1) lowInflight = w * inflightAtP25 lowTTFT = w * P25 + (1 - w) * P10Low effectiveTTFT = lowTTFT + (inflight - lowInflight) * (P50 - lowTTFT) / (inflightAtP50 - lowInflight) ``` -Clamped to `>= P10Low`. Under-observed models (not yet calibrated) seed at `P10Low`. -**Score:** +Clamped to `>= P10Low`. + +### Model states + +Each candidate is in one of three states, from its published metrics: + +- **cold** — `Floor() == 0` (never observed, or fewer than `minRequests` observations, so the floor + is not yet trustworthy). No operating point; seeded optimistically at the best observed TTFT. +- **seed** — has a floor but is not yet calibrated (`RecentN < minRequests`, or no inflight + operating point). Predicts at the floor. +- **trusted** — calibrated: `RecentN >= minRequests`, `inflightAtP50 > 0`, `P50 > floor`. Uses the + full blended prediction above. + +## Score + ``` score = (maxTTFT - effectiveTTFT) / (maxTTFT - minTTFT) ``` -Under-observed models (UNOBSERVED or SEED state) receive an optimistic high score. With -`explorationRate > 0`, that high score is suppressed to 0 with probability `(1 - explorationRate)` -so only ~`explorationRate` of requests probe the under-observed model for calibration. -## Why it works physically +Lowest predicted TTFT scores highest. Cold models seed at `minTTFT`; if every model is cold, all +score 1.0. + +### Exploration + +An under-observed pool can be starved: competing against a calibrated pool it may score low and +never win the traffic it needs to calibrate, so it stays under-observed forever. `explorationRate` +breaks that loop. Each under-observed pool is flipped **independently** per request: + +- **heads** (probability `explorationRate`): that pool's final score is forced to `1.0` — the top — + so the picker sends it a guaranteed calibration probe. +- **tails** (probability `1 - explorationRate`): that pool is suppressed to `0` so only calibrated + pools compete — but only when a calibrated pool exists to take the traffic. + +`explorationRate == 0` disables exploration (every request goes to the winner). The override is +applied to the **final score only** — it never feeds the min/max normalisation, so a probe cannot +distort the trusted pools' scores. + +Because each under-observed pool is flipped independently, every cold pool is probed at its own +`explorationRate` regardless of how many others are cold — so the fraction of requests that explore +*something* grows with the number of cold pools. +This trades a larger exploration budget for guaranteed per-pool probe coverage. + +Together with the extractor's floor sample guard, this reproduces automatically what a manual warmup +would do by hand: a brand-new pool reads as cold → receives probes → accumulates observations → +crosses `minRequests` → competes on its true latency. + +## Why the blend works physically When more requests are in flight, a new request waits longer in the queue, so TTFT rises with inflight — and it rises *faster than linearly* as the server approaches saturation (queueing). @@ -56,41 +78,30 @@ The scorer draws a line through two points it has actually observed: - median requests at higher load: `(inflightAtP50, P50)` Because both anchors sit *inside* the observed load cloud, the line follows the **local slope** of -that convex curve — so it does not systematically under-predict the way a single chord drawn from a +that convex curve — so it does not systematically under-predict the way a single chord from a synthetic zero-load floor does. At low load the two anchors collapse together (every request sees similar, low inflight), so their -slope is ill-defined. The blend weight `w` handles this smoothly: as the inflight gap shrinks, `w → -0` slides the low anchor down to `(0, P10Low)`, recovering the stable floor chord. There is no -threshold and no discontinuity — the prediction transitions continuously between the two regimes, -and the denominator `inflightAtP50 - w*inflightAtP25` can never collapse. The only knob, -`anchorGapScale`, is a numerical-conditioning scale (how much inflight separation counts as "well -separated"), not a fitted parameter. +slope is ill-defined. The blend weight `w` handles this smoothly: as the inflight gap shrinks, +`w → 0` slides the low anchor down to `(0, P10Low)`, recovering the stable floor chord. There is no +threshold and no discontinuity, and the denominator `inflightAtP50 - w*inflightAtP25` can never +collapse. The only knob, `anchorGapScale`, is a numerical-conditioning scale (how much inflight +separation counts as "well separated"), not a fitted parameter. ## Parameters -### Scorer (`ttft-aware-scorer`) - | Parameter | Default | Description | |---|---|---| -| `explorationRate` | 0.0 | Fraction of requests routed to under-observed models for calibration probing. 0 = all traffic to the trusted winner; 0.1 = ~10% probe. | +| `explorationRate` | 0.0 | Per-pool probability that an under-observed pool is probed on a given request (flipped independently per pool). 0 = all traffic to the trusted winner; 0.1 = each cold pool probed on ~10% of requests. | | `anchorGapScale` | 2.0 | Inflight separation (`inflightAtP50 - inflightAtP25`) at which the prediction fully trusts the in-cloud secant; below it the low anchor blends toward the floor chord. Must be > 0. | -### Extractor (`ttft-percentile-extractor`) - -| Parameter | Default | Description | -|---|---|---| -| `maxObservationAge` | 3m | Time bound for the short window (P25 / P50 / P10) | -| `maxRequests` | 100 | Cap the short window to the most recent N observations | -| `minRequests` | 10 | Minimum capped-window count before the scorer trusts the operating point | -| `windowSize` | 5000 | Ring buffer capacity (~200 KB per model) | -| `bucketDuration` | 1m | Window for each floor-history entry's P10; keep `<= maxObservationAge` | -| `bucketHistorySize` | 720 | Per-bucket P10s kept for the floor (`bucketDuration * bucketHistorySize` = horizon, 12h); min/max evicted when full | +The scorer also reads `minRequests` from the extractor's published metrics; it is configured on the +[extractor](../../../datalayer/ttftpercentile/README.md), not here. ## Example configuration -An end-to-end Helm values override wiring the scorer together with the TTFT extractor, -the model-config datasource, and a picker: +An end-to-end Helm values override wiring the scorer together with the TTFT extractor, the +model-config datasource, and a picker: ```yaml payloadProcessor: @@ -136,32 +147,18 @@ payloadProcessor: - pluginRef: model-config-datasource ``` -The scorer requires the `ttft-percentile-extractor` in `datalayer.extractors`, and a model -list (here via `model-config-datasource`) so model selection has candidates. +The scorer requires the `ttft-percentile-extractor` in `datalayer.extractors`, and a model list +(here via `model-config-datasource`) so model selection has candidates. -## Possible Enhancements +## Possible enhancement — score-proportional picker -### Score-proportional picker +`max-score-picker` sends 100% of traffic to the single winner, turning every small score difference +into a full traffic flip. This causes oscillation: the best model overloads, all traffic switches to +the other, the first model drains and wins again. `score-proportional-picker` eliminates this by +routing probabilistically: -`max-score-picker` sends 100% of traffic to the single winner, turning every small -score difference into a full traffic flip. This causes oscillation: the best model -overloads, all traffic switches to the other, the first model drains and wins again. -`score-proportional-picker` eliminates this by routing probabilistically: ``` P(model i) proportional to score_i^(1/T) # T = temperature, default 1.0 ``` -At T = 1.0, a model scoring 0.8 vs 0.2 receives ~80% vs 20% of requests. - -### Prompt-length-aware floor - -P10Low is estimated from the fastest observed completions, which tend to be short-prompt -requests. For a long-prompt request, the hardware-floor prefill time is intrinsically higher, -so the scorer under-predicts TTFT even at zero queue depth. -A more accurate floor would scale with the incoming prompt token count: -``` -P10Low(tokens) = base_prefill + tokens × prefill_rate -``` -where `base_prefill` and `prefill_rate` are fit from observations bucketed by prompt length. -This matters most when the workload has high prompt-length variance (e.g. RAG pipelines -mixing short queries with large context windows). +At T = 1.0, a model scoring 0.8 vs 0.2 receives ~80% vs 20% of requests. diff --git a/pkg/framework/plugins/modelselector/scorer/ttftaware/plugin.go b/pkg/framework/plugins/modelselector/scorer/ttftaware/plugin.go index 571623b9..a91aac06 100644 --- a/pkg/framework/plugins/modelselector/scorer/ttftaware/plugin.go +++ b/pkg/framework/plugins/modelselector/scorer/ttftaware/plugin.go @@ -121,12 +121,9 @@ type modelEval struct { } // Score ranks models by predicted TTFT: score = (maxTTFT − effectiveTTFT) / (maxTTFT − minTTFT). -// -// Cold models (no floor) are seeded at the best observed TTFT; if every model is cold, all -// score 1.0. With explorationRate > 0, each request makes one exploration decision: with -// probability explorationRate the under-observed models keep their scores (so one may win and -// get a calibration probe), otherwise they are suppressed to 0 — but only when a calibrated -// model exists to take the traffic. +// Cold models (no floor) are seeded at the best observed TTFT; if every model is cold, all score +// 1.0. With explorationRate > 0 each under-observed model is independently explored (forced to the +// top) or suppressed — see the scoring loop below and README.md. func (s *TTFTAwareScorer) Score(ctx context.Context, cycleState *plugin.CycleState, _ *requesthandling.InferenceRequest, models []datalayer.Model) map[datalayer.Model]float64 { evals := make(map[datalayer.Model]*modelEval, len(models)) minEff := math.MaxFloat64 @@ -158,12 +155,13 @@ func (s *TTFTAwareScorer) Score(ctx context.Context, cycleState *plugin.CycleSta return scores } - // One exploration decision per request: with probability (1 - explorationRate) suppress all - // under-observed models so only calibrated models compete; otherwise leave them scored so an - // under-observed model can win and get a calibration probe. Only suppress when a calibrated - // model exists to take the traffic. - suppressUntrusted := s.explorationRate > 0 && anyTrusted && rand.Float64() >= s.explorationRate - + // Independent coin per under-observed model (explorationRate == 0 disables exploration). Heads + // forces that model to the top so the picker sends it a calibration probe; tails suppresses it to + // 0 so only calibrated models compete, but only when a calibrated model exists to take over. Each + // under-observed model is flipped independently, so the fraction of requests that explore grows + // with the number of under-observed models (~1-(1-rate)^k) — a larger budget in exchange for + // guaranteed per-model probe coverage. The override sets the final score only — eff and the + // min/max normalization above are untouched. for _, model := range models { e := evals[model] if !e.observed { @@ -174,8 +172,12 @@ func (s *TTFTAwareScorer) Score(ctx context.Context, cycleState *plugin.CycleSta } else { scores[model] = (maxEff - e.eff) / (maxEff - minEff) } - if suppressUntrusted && !e.trusted { - scores[model] = 0 + if s.explorationRate > 0 && !e.trusted { + if rand.Float64() < s.explorationRate { + scores[model] = 1.0 // explore: probe this under-observed model + } else if anyTrusted { + scores[model] = 0 // exploit: a calibrated model takes the traffic + } } } From 1489e721fbca371b9cfcb6854ef95296e8e4f4e5 Mon Sep 17 00:00:00 2001 From: Mohammad Date: Sat, 25 Jul 2026 22:03:07 +0300 Subject: [PATCH 25/27] Cleanup. Signed-off-by: Mohammad --- .../modelselector/scorer/ttftaware/plugin.go | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/pkg/framework/plugins/modelselector/scorer/ttftaware/plugin.go b/pkg/framework/plugins/modelselector/scorer/ttftaware/plugin.go index a91aac06..8cf1d9f3 100644 --- a/pkg/framework/plugins/modelselector/scorer/ttftaware/plugin.go +++ b/pkg/framework/plugins/modelselector/scorer/ttftaware/plugin.go @@ -114,27 +114,25 @@ func (s *TTFTAwareScorer) WithAnchorGapScale(g float64) *TTFTAwareScorer { // modelEval is the scorer's per-model working state for one Score call. type modelEval struct { - metrics ttftpercentile.TTFTPercentileMetrics eff float64 trusted bool // calibrated operating point observed bool // has a service floor (not truly cold) } -// Score ranks models by predicted TTFT: score = (maxTTFT − effectiveTTFT) / (maxTTFT − minTTFT). +// Score ranks models by predicted TTFT: score = (maxTTFT - effectiveTTFT) / (maxTTFT - minTTFT). // Cold models (no floor) are seeded at the best observed TTFT; if every model is cold, all score // 1.0. With explorationRate > 0 each under-observed model is independently explored (forced to the // top) or suppressed — see the scoring loop below and README.md. -func (s *TTFTAwareScorer) Score(ctx context.Context, cycleState *plugin.CycleState, _ *requesthandling.InferenceRequest, models []datalayer.Model) map[datalayer.Model]float64 { - evals := make(map[datalayer.Model]*modelEval, len(models)) +func (s *TTFTAwareScorer) Score(ctx context.Context, _ *plugin.CycleState, _ *requesthandling.InferenceRequest, models []datalayer.Model) map[datalayer.Model]float64 { + evals := make([]modelEval, len(models)) minEff := math.MaxFloat64 maxEff := 0.0 anyObserved := false anyTrusted := false - for _, model := range models { - m := metricsFor(model) - eff, trusted, observed := s.predict(m) - evals[model] = &modelEval{metrics: m, eff: eff, trusted: trusted, observed: observed} + for i, model := range models { + eff, trusted, observed := s.predict(metricsFor(model)) + evals[i] = modelEval{eff: eff, trusted: trusted, observed: observed} if observed { anyObserved = true minEff = min(minEff, eff) @@ -162,8 +160,8 @@ func (s *TTFTAwareScorer) Score(ctx context.Context, cycleState *plugin.CycleSta // with the number of under-observed models (~1-(1-rate)^k) — a larger budget in exchange for // guaranteed per-model probe coverage. The override sets the final score only — eff and the // min/max normalization above are untouched. - for _, model := range models { - e := evals[model] + for i, model := range models { + e := &evals[i] if !e.observed { e.eff = minEff // seed cold models at the best observed TTFT (optimistic) } @@ -182,8 +180,8 @@ func (s *TTFTAwareScorer) Score(ctx context.Context, cycleState *plugin.CycleSta } if dl := log.FromContext(ctx).V(logutil.DEBUG); dl.Enabled() { - for _, model := range models { - e := evals[model] + for i, model := range models { + e := evals[i] dl.Info("ttft-aware score", "model", model.GetName(), "effectiveTTFT", e.eff, "score", scores[model], "trusted", e.trusted) } From 4c9fa404002e7a711d5e0d189572423b493c3386 Mon Sep 17 00:00:00 2001 From: Mohammad Date: Thu, 30 Jul 2026 15:52:30 +0300 Subject: [PATCH 26/27] Fix underprediction when the inflight less that lower. Signed-off-by: Mohammad --- .../modelselector/scorer/ttftaware/README.md | 35 +++++++++++++------ .../modelselector/scorer/ttftaware/plugin.go | 27 +++++++++----- 2 files changed, 44 insertions(+), 18 deletions(-) diff --git a/pkg/framework/plugins/modelselector/scorer/ttftaware/README.md b/pkg/framework/plugins/modelselector/scorer/ttftaware/README.md index 53838d21..eaf35878 100644 --- a/pkg/framework/plugins/modelselector/scorer/ttftaware/README.md +++ b/pkg/framework/plugins/modelselector/scorer/ttftaware/README.md @@ -4,24 +4,39 @@ Routes each request to the model with the lowest predicted TTFT under current lo It consumes the per-model snapshot published by the [TTFT percentile extractor](../../../datalayer/ttftpercentile/README.md) — the service floor -`P10Low`, the operating points `P25` / `P50`, and their banded inflight averages `inflightAtP25` / -`inflightAtP50` — and turns them into a prediction and a score. See the extractor README for how +`P10Low`, the low/high operating points `LowTTFT` / `HighTTFT`, and their banded inflight averages +`InflightAtLow` / `InflightAtHigh` — and turns them into a prediction and a score. The two operating +percentiles default to **P25 / P50** and are configured on the extractor (`lowPercentile` / +`highPercentile`); this README uses P25 / P50 for the defaults. See the extractor README for how those inputs are measured. ## Prediction — `effectiveTTFT` -The predicted TTFT for a request arriving now is a line through the high operating point -`(inflightAtP50, P50)` and a low anchor that blends between the in-cloud point -`(inflightAtP25, P25)` and the load-free floor `(0, P10Low)`: +The predicted TTFT for a request arriving now is a **two-segment** piecewise line. The upper segment +is the in-cloud secant through the low anchor and the high operating point `(inflightAtP50, P50)`; the +low anchor itself blends between the in-cloud point `(inflightAtP25, P25)` and the load-free floor +`(0, P10Low)`: ``` -w = clamp((inflightAtP50 - inflightAtP25) / anchorGapScale, 0, 1) -lowInflight = w * inflightAtP25 -lowTTFT = w * P25 + (1 - w) * P10Low -effectiveTTFT = lowTTFT + (inflight - lowInflight) * (P50 - lowTTFT) / (inflightAtP50 - lowInflight) +w = clamp((inflightAtP50 - inflightAtP25) / anchorGapScale, 0, 1) +lowInflight = w * inflightAtP25 +lowTTFT = w * P25 + (1 - w) * P10Low + +if inflight >= lowInflight: # upper segment: in-cloud secant (unchanged) + effectiveTTFT = lowTTFT + (inflight - lowInflight) * (P50 - lowTTFT) / (inflightAtP50 - lowInflight) +else: # lower segment: (0, P10Low) -> (lowInflight, lowTTFT) + effectiveTTFT = P10Low + inflight * (lowTTFT - P10Low) / lowInflight ``` -Clamped to `>= P10Low`. +Clamped to `>= P10Low` (now redundant, kept as a guard). The two segments meet at `lowInflight` +(both give `lowTTFT`), and the lower one hits `P10Low` at zero load. + +**Why two segments:** the secant's slope is measured *between* the anchors — in the high-load region +where TTFT is dominated by queueing, so it is steep. Extrapolating it **backwards** below the low +anchor (when the current inflight drops below `lowInflight`) makes TTFT fall far too fast, dive below +the floor within a few requests, and pin a draining-but-still-loaded pool at its idle prefill time — +which then wins the route, refills, and oscillates. The lower segment instead rises from the floor at +zero load up to the low anchor, matching how TTFT flattens toward the floor as the queue drains. ### Model states diff --git a/pkg/framework/plugins/modelselector/scorer/ttftaware/plugin.go b/pkg/framework/plugins/modelselector/scorer/ttftaware/plugin.go index 8cf1d9f3..167098cc 100644 --- a/pkg/framework/plugins/modelselector/scorer/ttftaware/plugin.go +++ b/pkg/framework/plugins/modelselector/scorer/ttftaware/plugin.go @@ -15,8 +15,9 @@ limitations under the License. */ // Package ttftaware routes each request to the model with the lowest predicted TTFT under -// current load. Prediction is a line through (inflightAtP50, P50) and a low anchor blended -// between the in-cloud point (inflightAtP25, P25) and the floor (0, P10Low). See README.md +// current load. Prediction is a two-segment line: an in-cloud secant to (inflightAtP50, P50) at and +// above the low anchor, and a segment from the floor (0, P10Low) up to the low anchor below it. The +// low anchor blends between the in-cloud point (inflightAtP25, P25) and the floor. See README.md // for the equations and rationale. package ttftaware @@ -198,20 +199,30 @@ func (s *TTFTAwareScorer) predict(m ttftpercentile.TTFTPercentileMetrics) (eff f if floor == 0 { return 0, false, false } - if !(m.RecentN >= m.MinRequests && m.InflightAtP50 > 0 && m.P50TTFT > floor) { + if !(m.RecentN >= m.MinRequests && m.InflightAtHigh > 0 && m.HighTTFT > floor) { return floor, false, true // optimistic seed at the floor } // Blend the low anchor between (iP25, P25) and (0, floor) by the anchor separation; w→1 // under load gives the in-cloud secant, w→0 at low load gives the floor chord. - w := max(0, min(1, (m.InflightAtP50-m.InflightAtP25)/s.anchorGapScale)) - lowInflight := w * m.InflightAtP25 - lowTTFT := w*m.P25TTFT + (1-w)*floor + w := max(0, min(1, (m.InflightAtHigh-m.InflightAtLow)/s.anchorGapScale)) + lowInflight := w * m.InflightAtLow + lowTTFT := w*m.LowTTFT + (1-w)*floor cur := float64(m.Requests) - eff = lowTTFT + (cur-lowInflight)*(m.P50TTFT-lowTTFT)/(m.InflightAtP50-lowInflight) + if cur < lowInflight && lowInflight > 0 { + // Below the low anchor, interpolate the low segment (0, floor) -> (lowInflight, lowTTFT) + // instead of extrapolating the steep queueing secant. The secant's slope was measured in the + // high-load region and, run backwards, dives below the floor within a few requests — pinning a + // draining-but-still-loaded pool at its idle latency. TTFT flattens toward the floor as the + // queue drains, so this segment tracks that. + eff = floor + cur*(lowTTFT-floor)/lowInflight + } else { + // At/above the low anchor: the in-cloud secant (lowInflight, lowTTFT) -> (InflightAtHigh, P50). + eff = lowTTFT + (cur-lowInflight)*(m.HighTTFT-lowTTFT)/(m.InflightAtHigh-lowInflight) + } if eff < floor { - eff = floor + eff = floor // mathematically redundant now; kept as a defensive guard } return eff, true, true } From 45033a482c6daf3b13acfa6f9c2ced52a4ac040f Mon Sep 17 00:00:00 2001 From: Mohammad Date: Sun, 2 Aug 2026 00:50:27 +0300 Subject: [PATCH 27/27] Update Readme.md. Signed-off-by: Mohammad --- .../modelselector/scorer/ttftaware/README.md | 220 +++++++----------- .../modelselector/scorer/ttftaware/plugin.go | 156 +++++++------ 2 files changed, 162 insertions(+), 214 deletions(-) diff --git a/pkg/framework/plugins/modelselector/scorer/ttftaware/README.md b/pkg/framework/plugins/modelselector/scorer/ttftaware/README.md index eaf35878..5479ed0a 100644 --- a/pkg/framework/plugins/modelselector/scorer/ttftaware/README.md +++ b/pkg/framework/plugins/modelselector/scorer/ttftaware/README.md @@ -4,55 +4,80 @@ Routes each request to the model with the lowest predicted TTFT under current lo It consumes the per-model snapshot published by the [TTFT percentile extractor](../../../datalayer/ttftpercentile/README.md) — the service floor -`P10Low`, the low/high operating points `LowTTFT` / `HighTTFT`, and their banded inflight averages -`InflightAtLow` / `InflightAtHigh` — and turns them into a prediction and a score. The two operating -percentiles default to **P25 / P50** and are configured on the extractor (`lowPercentile` / -`highPercentile`); this README uses P25 / P50 for the defaults. See the extractor README for how -those inputs are measured. +`P10Low`, the operating points `LowTTFT` / `HighTTFT` (default P25 / P50, configured on the +extractor) and their banded inflight averages `InflightAtLow` / `InflightAtHigh` — and turns them +into a prediction and a score. -## Prediction — `effectiveTTFT` +## Prediction — `predictedTTFT` -The predicted TTFT for a request arriving now is a **two-segment** piecewise line. The upper segment -is the in-cloud secant through the low anchor and the high operating point `(inflightAtP50, P50)`; the -low anchor itself blends between the in-cloud point `(inflightAtP25, P25)` and the load-free floor -`(0, P10Low)`: +Every pool has a latency curve: how long a new request waits as a function of how many requests are +already in flight. We measure three points on it and interpolate. + +| point | coordinates | meaning | +|---|---|---| +| **A** | `(0, P10Low)` | queue-free service time | +| **B** | `(InflightAtLow, LowTTFT)` | low operating point (default P25) | +| **C** | `(InflightAtHigh, HighTTFT)` | high operating point (default P50) | + +**A** and **C** always define the curve. **B** is inserted between them only when +[admissible](#when-the-low-point-is-used), splitting it into two segments. Any single prediction +reads one segment, so it uses two of the three points. Every point is something the extractor observed. ``` -w = clamp((inflightAtP50 - inflightAtP25) / anchorGapScale, 0, 1) -lowInflight = w * inflightAtP25 -lowTTFT = w * P25 + (1 - w) * P10Low - -if inflight >= lowInflight: # upper segment: in-cloud secant (unchanged) - effectiveTTFT = lowTTFT + (inflight - lowInflight) * (P50 - lowTTFT) / (inflightAtP50 - lowInflight) -else: # lower segment: (0, P10Low) -> (lowInflight, lowTTFT) - effectiveTTFT = P10Low + inflight * (lowTTFT - P10Low) / lowInflight +if B admissible: + if inflight < InflightAtLow: # segment A->B + predictedTTFT = P10Low + inflight * (LowTTFT - P10Low) / InflightAtLow + else: # segment B->C, extended past C + predictedTTFT = LowTTFT + (inflight - InflightAtLow) * + (HighTTFT - LowTTFT) / (InflightAtHigh - InflightAtLow) +else: # segment A->C, extended past C + predictedTTFT = P10Low + inflight * (HighTTFT - P10Low) / InflightAtHigh ``` -Clamped to `>= P10Low` (now redundant, kept as a guard). The two segments meet at `lowInflight` -(both give `lowTTFT`), and the lower one hits `P10Low` at zero load. +The curve is continuous (both branches give `LowTTFT` at `InflightAtLow`), equals `P10Low` at zero +load, and is monotone non-decreasing given `P10Low < LowTTFT < HighTTFT` — which the admissibility +checks enforce. `predictedTTFT` is clamped to `>= P10Low` as a defensive guard. -**Why two segments:** the secant's slope is measured *between* the anchors — in the high-load region -where TTFT is dominated by queueing, so it is steep. Extrapolating it **backwards** below the low -anchor (when the current inflight drops below `lowInflight`) makes TTFT fall far too fast, dive below -the floor within a few requests, and pin a draining-but-still-loaded pool at its idle prefill time — -which then wins the route, refills, and oscillates. The lower segment instead rises from the floor at -zero load up to the low anchor, matching how TTFT flattens toward the floor as the queue drains. +**Why three points and not two.** TTFT rises *faster than linearly* with inflight as a pool +approaches saturation. A single chord from the floor to **C** cuts across that convex curve and +under-predicts in between; **B** lets the curve bend so the loaded segment follows the local slope +where the pool is actually operating. -### Model states +**Why A→B is a segment of its own.** The B→C slope is measured where queueing dominates, so it is +steep. Running it backwards below **B** makes TTFT cross below the floor within a handful of +requests — a draining-but-still-loaded pool would be predicted at its idle latency and win every +decision it appeared in. Interpolating from `(0, floor)` matches how TTFT flattens as the queue +drains. + +### When the low point is used + +**B** is admissible only when all of these hold. They are conditions on a measured point, not +tuning knobs: + +| check | condition | why | +|---|---|---| +| separated in load | `InflightAtHigh - InflightAtLow >= minInflightGap` | the B→C slope is `ΔTTFT / Δinflight`; if both points sit at the same load that denominator is noise and the slope is meaningless | +| ordered in latency | `HighTTFT > LowTTFT` | TTFT must rise with the percentile. If noise inverts them the slope goes negative and the *most* loaded pool scores best, feeding a saturated pool | +| above the floor | `LowTTFT > P10Low` | `P10Low` is a long-window statistic and `LowTTFT` a recent one, so after a drain the recent P25 can fall below it — which would tilt the A→B segment downwards | +| positive inflight | `InflightAtLow > 0` | keeps the A→B divisor safe; the gap check alone does not imply it | + +When **B** is dropped the curve is the single floor chord **A → C**, which is well defined at any +load. -Each candidate is in one of three states, from its published metrics: -- **cold** — `Floor() == 0` (never observed, or fewer than `minRequests` observations, so the floor - is not yet trustworthy). No operating point; seeded optimistically at the best observed TTFT. -- **seed** — has a floor but is not yet calibrated (`RecentN < minRequests`, or no inflight - operating point). Predicts at the floor. -- **trusted** — calibrated: `RecentN >= minRequests`, `inflightAtP50 > 0`, `P50 > floor`. Uses the - full blended prediction above. +### Model states + +- **cold** — `Floor() == 0` (never observed, or fewer than `minRequests` observations so the floor + is not yet trustworthy). Seeded optimistically at the best observed TTFT. +- **seed** — has a floor but is not calibrated (`RecentN < minRequests`, or no inflight operating + point). Predicts at the floor. +- **trusted** — `RecentN >= minRequests`, `InflightAtHigh > 0`, `HighTTFT > floor`. Uses the curve + above. ## Score ``` -score = (maxTTFT - effectiveTTFT) / (maxTTFT - minTTFT) +score = (maxTTFT - predictedTTFT) / (maxTTFT - minTTFT) ``` Lowest predicted TTFT scores highest. Cold models seed at `minTTFT`; if every model is cold, all @@ -60,120 +85,35 @@ score 1.0. ### Exploration -An under-observed pool can be starved: competing against a calibrated pool it may score low and -never win the traffic it needs to calibrate, so it stays under-observed forever. `explorationRate` -breaks that loop. Each under-observed pool is flipped **independently** per request: - -- **heads** (probability `explorationRate`): that pool's final score is forced to `1.0` — the top — - so the picker sends it a guaranteed calibration probe. -- **tails** (probability `1 - explorationRate`): that pool is suppressed to `0` so only calibrated - pools compete — but only when a calibrated pool exists to take the traffic. - -`explorationRate == 0` disables exploration (every request goes to the winner). The override is -applied to the **final score only** — it never feeds the min/max normalisation, so a probe cannot -distort the trusted pools' scores. - -Because each under-observed pool is flipped independently, every cold pool is probed at its own -`explorationRate` regardless of how many others are cold — so the fraction of requests that explore -*something* grows with the number of cold pools. -This trades a larger exploration budget for guaranteed per-pool probe coverage. +An under-observed pool can be starved: competing against a calibrated pool it may never win the +traffic it needs to calibrate. `explorationRate` breaks that loop — each under-observed pool is +flipped independently per request, and with probability `explorationRate` its final score is forced +to `1.0` so the picker sends it a probe; otherwise it is suppressed to `0`, but only when a +calibrated pool exists to take the traffic. The override applies to the **final score only**, so a +probe never distorts the trusted pools' normalisation. -Together with the extractor's floor sample guard, this reproduces automatically what a manual warmup -would do by hand: a brand-new pool reads as cold → receives probes → accumulates observations → -crosses `minRequests` → competes on its true latency. - -## Why the blend works physically - -When more requests are in flight, a new request waits longer in the queue, so TTFT rises with -inflight — and it rises *faster than linearly* as the server approaches saturation (queueing). - -The scorer draws a line through two points it has actually observed: - -- fast requests ran at lower load: `(inflightAtP25, P25)` -- median requests at higher load: `(inflightAtP50, P50)` - -Because both anchors sit *inside* the observed load cloud, the line follows the **local slope** of -that convex curve — so it does not systematically under-predict the way a single chord from a -synthetic zero-load floor does. - -At low load the two anchors collapse together (every request sees similar, low inflight), so their -slope is ill-defined. The blend weight `w` handles this smoothly: as the inflight gap shrinks, -`w → 0` slides the low anchor down to `(0, P10Low)`, recovering the stable floor chord. There is no -threshold and no discontinuity, and the denominator `inflightAtP50 - w*inflightAtP25` can never -collapse. The only knob, `anchorGapScale`, is a numerical-conditioning scale (how much inflight -separation counts as "well separated"), not a fitted parameter. +Together with the extractor's floor sample guard this reproduces what a manual warmup would do: a +new pool reads as cold → receives probes → crosses `minRequests` → competes on its true latency. ## Parameters | Parameter | Default | Description | |---|---|---| -| `explorationRate` | 0.0 | Per-pool probability that an under-observed pool is probed on a given request (flipped independently per pool). 0 = all traffic to the trusted winner; 0.1 = each cold pool probed on ~10% of requests. | -| `anchorGapScale` | 2.0 | Inflight separation (`inflightAtP50 - inflightAtP25`) at which the prediction fully trusts the in-cloud secant; below it the low anchor blends toward the floor chord. Must be > 0. | - -The scorer also reads `minRequests` from the extractor's published metrics; it is configured on the -[extractor](../../../datalayer/ttftpercentile/README.md), not here. +| `explorationRate` | 0.0 | Per-pool probability that an under-observed pool is probed on a given request. 0 = all traffic to the trusted winner. | +| `minInflightGap` | 2.0 | Minimum inflight separation between the operating points for the low one to be used as an anchor. Must be > 0. | +| `roundTTFTStep` | 0.0 | Quantize each prediction to a multiple of this many seconds before ranking (e.g. `0.01` = 10 ms). Pools landing in the same bucket tie and the picker splits them, instead of one winning on a difference too small to be meaningful. `0` = disabled. Must be >= 0. | -## Example configuration +`minRequests` is read from the extractor's published metrics and configured +[there](../../../datalayer/ttftpercentile/README.md), not here. -An end-to-end Helm values override wiring the scorer together with the TTFT extractor, the -model-config datasource, and a picker: +## Configuration ```yaml -payloadProcessor: - customConfig: - plugins: - - type: body-field-to-header - parameters: - fieldName: model - headerName: X-Gateway-Model-Name - - type: base-model-to-header - - type: model-selector - - type: ttft-aware-scorer - parameters: - explorationRate: 0.1 # 10% of requests probe under-observed models; 0 = disabled - anchorGapScale: 2.0 # inflight separation at which the in-cloud secant is fully trusted - - type: max-score-picker - - type: ttft-percentile-extractor - parameters: - intervalDuration: 1s - windowSize: 5000 - maxObservationAge: 3m - maxRequests: 100 - minRequests: 20 - bucketDuration: 1m - bucketHistorySize: 720 - - type: model-config-datasource - parameters: - modelsPath: /config/models.json - profiles: - - name: default - plugins: - request: - - pluginRef: model-selector - - pluginRef: ttft-aware-scorer - weight: 1.0 - - pluginRef: max-score-picker - - pluginRef: body-field-to-header - - pluginRef: base-model-to-header - datalayer: - extractors: - - pluginRef: ttft-percentile-extractor - datasources: - - pluginRef: model-config-datasource -``` - -The scorer requires the `ttft-percentile-extractor` in `datalayer.extractors`, and a model list -(here via `model-config-datasource`) so model selection has candidates. - -## Possible enhancement — score-proportional picker - -`max-score-picker` sends 100% of traffic to the single winner, turning every small score difference -into a full traffic flip. This causes oscillation: the best model overloads, all traffic switches to -the other, the first model drains and wins again. `score-proportional-picker` eliminates this by -routing probabilistically: - -``` -P(model i) proportional to score_i^(1/T) # T = temperature, default 1.0 +- type: ttft-aware-scorer + parameters: + explorationRate: 0.1 + minInflightGap: 2.0 ``` -At T = 1.0, a model scoring 0.8 vs 0.2 receives ~80% vs 20% of requests. +The scorer requires `ttft-percentile-extractor` in `datalayer.extractors`, a model list (e.g. via +`model-config-datasource`) so model selection has candidates, and a picker. diff --git a/pkg/framework/plugins/modelselector/scorer/ttftaware/plugin.go b/pkg/framework/plugins/modelselector/scorer/ttftaware/plugin.go index 167098cc..42365c7e 100644 --- a/pkg/framework/plugins/modelselector/scorer/ttftaware/plugin.go +++ b/pkg/framework/plugins/modelselector/scorer/ttftaware/plugin.go @@ -14,11 +14,9 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Package ttftaware routes each request to the model with the lowest predicted TTFT under -// current load. Prediction is a two-segment line: an in-cloud secant to (inflightAtP50, P50) at and -// above the low anchor, and a segment from the floor (0, P10Low) up to the low anchor below it. The -// low anchor blends between the in-cloud point (inflightAtP25, P25) and the floor. See README.md -// for the equations and rationale. +// Package ttftaware routes each request to the model with the lowest predicted TTFT under current +// load. The prediction is a piecewise-linear curve through the model's measured points, evaluated +// at its current inflight count. See README.md for the model, the equations and the rationale. package ttftaware import ( @@ -41,35 +39,39 @@ import ( const ( PluginType = "ttft-aware-scorer" - defaultExplorationRate = 0.0 // off by default; set e.g. 0.1 for 10% exploration - defaultAnchorGapScale = 2.0 // inflight separation at which the blend fully trusts the secant + defaultExplorationRate = 0.0 // disabled; 0.1 probes each under-observed model on ~10% of requests + defaultMinInflightGap = 2.0 // inflight separation the operating points need to define a slope + defaultRoundTTFTStep = 0.0 // disabled; e.g. 0.01 quantizes predictions to 10ms ) var _ modelselector.Scorer = &TTFTAwareScorer{} -// TTFTAwareScorerConfig holds optional parameters for the scorer plugin. +// TTFTAwareScorerConfig holds optional parameters for the scorer plugin. See README.md. type TTFTAwareScorerConfig struct { - // ExplorationRate controls the probability of routing a request to an under-observed - // model (UNOBSERVED or SEED state) instead of the trusted best model. A value of 0.1 - // means ~10% of requests probe the under-observed model, preventing a burst of traffic - // before the first responses return and P50 is calibrated. - // Range [0, 1]. Default 0 (disabled — every request goes to the winner). + // ExplorationRate is the per-model probability that an under-observed model is forced to the + // top score, so it gets the traffic it needs to calibrate. Range [0, 1]. Default 0 (disabled). ExplorationRate float64 `json:"explorationRate,omitempty"` - // AnchorGapScale is the inflight separation at which the prediction fully trusts the in-cloud - // secant; below it the low anchor blends toward the floor chord. Must be > 0. Default 2. - AnchorGapScale float64 `json:"anchorGapScale,omitempty"` + // MinInflightGap is the minimum inflight separation between the two operating points for the + // low one to be usable as a curve anchor. Below it their slope is noise. Must be > 0. + MinInflightGap float64 `json:"minInflightGap,omitempty"` + // RoundTTFTStep quantizes each prediction to a multiple of this many seconds before ranking + // (e.g. 0.01 = 10ms), so models closer together than the step tie and the picker splits them + // instead of one winning on a difference too small to be meaningful. Must be >= 0. Default 0. + RoundTTFTStep float64 `json:"roundTTFTStep,omitempty"` } type TTFTAwareScorer struct { typedName plugin.TypedName explorationRate float64 - anchorGapScale float64 + minInflightGap float64 + roundTTFTStep float64 } func ScorerFactory(name string, parameters json.RawMessage, _ plugin.Handle) (plugin.Plugin, error) { cfg := TTFTAwareScorerConfig{ ExplorationRate: defaultExplorationRate, - AnchorGapScale: defaultAnchorGapScale, + MinInflightGap: defaultMinInflightGap, + RoundTTFTStep: defaultRoundTTFTStep, } if len(parameters) > 0 { if err := json.Unmarshal(parameters, &cfg); err != nil { @@ -79,20 +81,25 @@ func ScorerFactory(name string, parameters json.RawMessage, _ plugin.Handle) (pl if cfg.ExplorationRate < 0 || cfg.ExplorationRate > 1 { return nil, fmt.Errorf("explorationRate must be in [0, 1] for plugin %q", name) } - if cfg.AnchorGapScale <= 0 { - return nil, fmt.Errorf("anchorGapScale must be > 0 for plugin %q", name) + if cfg.MinInflightGap <= 0 { + return nil, fmt.Errorf("minInflightGap must be > 0 for plugin %q", name) + } + if cfg.RoundTTFTStep < 0 { + return nil, fmt.Errorf("roundTTFTStep must be >= 0 for plugin %q", name) } return NewTTFTAwareScorer(). WithName(name). WithExplorationRate(cfg.ExplorationRate). - WithAnchorGapScale(cfg.AnchorGapScale), nil + WithMinInflightGap(cfg.MinInflightGap). + WithRoundTTFTStep(cfg.RoundTTFTStep), nil } func NewTTFTAwareScorer() *TTFTAwareScorer { return &TTFTAwareScorer{ typedName: plugin.TypedName{Type: PluginType, Name: PluginType}, explorationRate: defaultExplorationRate, - anchorGapScale: defaultAnchorGapScale, + minInflightGap: defaultMinInflightGap, + roundTTFTStep: defaultRoundTTFTStep, } } @@ -108,36 +115,43 @@ func (s *TTFTAwareScorer) WithExplorationRate(r float64) *TTFTAwareScorer { return s } -func (s *TTFTAwareScorer) WithAnchorGapScale(g float64) *TTFTAwareScorer { - s.anchorGapScale = g +func (s *TTFTAwareScorer) WithMinInflightGap(g float64) *TTFTAwareScorer { + s.minInflightGap = g + return s +} + +func (s *TTFTAwareScorer) WithRoundTTFTStep(step float64) *TTFTAwareScorer { + s.roundTTFTStep = step return s } // modelEval is the scorer's per-model working state for one Score call. type modelEval struct { - eff float64 + pred float64 trusted bool // calibrated operating point observed bool // has a service floor (not truly cold) } -// Score ranks models by predicted TTFT: score = (maxTTFT - effectiveTTFT) / (maxTTFT - minTTFT). -// Cold models (no floor) are seeded at the best observed TTFT; if every model is cold, all score -// 1.0. With explorationRate > 0 each under-observed model is independently explored (forced to the -// top) or suppressed — see the scoring loop below and README.md. +// Score ranks models by predicted TTFT: score = (maxTTFT - predictedTTFT) / (maxTTFT - minTTFT). +// Cold models are seeded at the best observed TTFT; if every model is cold, all score 1.0. func (s *TTFTAwareScorer) Score(ctx context.Context, _ *plugin.CycleState, _ *requesthandling.InferenceRequest, models []datalayer.Model) map[datalayer.Model]float64 { evals := make([]modelEval, len(models)) - minEff := math.MaxFloat64 - maxEff := 0.0 + minPred := math.MaxFloat64 + maxPred := 0.0 anyObserved := false anyTrusted := false for i, model := range models { - eff, trusted, observed := s.predict(metricsFor(model)) - evals[i] = modelEval{eff: eff, trusted: trusted, observed: observed} + pred, trusted, observed := s.predict(metricsFor(model)) + if s.roundTTFTStep > 0 { + // Quantize before ranking so predictions closer together than the step tie. + pred = math.Round(pred/s.roundTTFTStep) * s.roundTTFTStep + } + evals[i] = modelEval{pred: pred, trusted: trusted, observed: observed} if observed { anyObserved = true - minEff = min(minEff, eff) - maxEff = max(maxEff, eff) // cold models seed to minEff, so they can never be the max + minPred = min(minPred, pred) + maxPred = max(maxPred, pred) // cold models seed to minPred, so they can never be the max } if trusted { anyTrusted = true @@ -146,7 +160,7 @@ func (s *TTFTAwareScorer) Score(ctx context.Context, _ *plugin.CycleState, _ *re scores := make(map[datalayer.Model]float64, len(models)) - // No model has a floor yet → nothing to rank; explore all equally. + // No model has a floor yet -> nothing to rank; explore all equally. if !anyObserved { for _, model := range models { scores[model] = 1.0 @@ -154,77 +168,71 @@ func (s *TTFTAwareScorer) Score(ctx context.Context, _ *plugin.CycleState, _ *re return scores } - // Independent coin per under-observed model (explorationRate == 0 disables exploration). Heads - // forces that model to the top so the picker sends it a calibration probe; tails suppresses it to - // 0 so only calibrated models compete, but only when a calibrated model exists to take over. Each - // under-observed model is flipped independently, so the fraction of requests that explore grows - // with the number of under-observed models (~1-(1-rate)^k) — a larger budget in exchange for - // guaranteed per-model probe coverage. The override sets the final score only — eff and the - // min/max normalization above are untouched. for i, model := range models { e := &evals[i] if !e.observed { - e.eff = minEff // seed cold models at the best observed TTFT (optimistic) + e.pred = minPred // seed cold models at the best observed TTFT (optimistic) } - if maxEff == minEff { + if maxPred == minPred { scores[model] = 1.0 } else { - scores[model] = (maxEff - e.eff) / (maxEff - minEff) + scores[model] = (maxPred - e.pred) / (maxPred - minPred) } + // Independent coin per under-observed model: heads forces a calibration probe, tails + // suppresses it so calibrated models take the traffic. Final score only — never feeds the + // normalization above, so a probe cannot distort the trusted models' scores. if s.explorationRate > 0 && !e.trusted { if rand.Float64() < s.explorationRate { - scores[model] = 1.0 // explore: probe this under-observed model + scores[model] = 1.0 } else if anyTrusted { - scores[model] = 0 // exploit: a calibrated model takes the traffic + scores[model] = 0 } } } if dl := log.FromContext(ctx).V(logutil.DEBUG); dl.Enabled() { for i, model := range models { - e := evals[i] dl.Info("ttft-aware score", "model", model.GetName(), - "effectiveTTFT", e.eff, "score", scores[model], "trusted", e.trusted) + "predictedTTFT", evals[i].pred, "score", scores[model], "trusted", evals[i].trusted) } } return scores } -// predict returns the model's effective TTFT and whether its operating point is trusted -// (calibrated) and observed (has a floor). Cold models return (0, false, false); observed but -// uncalibrated models seed at the floor. See README.md for the blended-secant equation. -func (s *TTFTAwareScorer) predict(m ttftpercentile.TTFTPercentileMetrics) (eff float64, trusted, observed bool) { +// predict returns the model's predicted TTFT at its current inflight, and whether it is trusted +// (calibrated) and observed (has a floor). Cold -> (0, false, false); observed but uncalibrated -> +// the floor as an optimistic seed. See README.md for the curve and the admissibility checks. +func (s *TTFTAwareScorer) predict(m ttftpercentile.TTFTPercentileMetrics) (pred float64, trusted, observed bool) { floor := m.Floor() if floor == 0 { return 0, false, false } if !(m.RecentN >= m.MinRequests && m.InflightAtHigh > 0 && m.HighTTFT > floor) { - return floor, false, true // optimistic seed at the floor + return floor, false, true } - // Blend the low anchor between (iP25, P25) and (0, floor) by the anchor separation; w→1 - // under load gives the in-cloud secant, w→0 at low load gives the floor chord. - w := max(0, min(1, (m.InflightAtHigh-m.InflightAtLow)/s.anchorGapScale)) - lowInflight := w * m.InflightAtLow - lowTTFT := w*m.LowTTFT + (1-w)*floor + // The low point is admissible only if the two operating points are separated in load, ordered + // in latency, and above the floor — otherwise its segment slopes on noise or downwards. + useLow := m.InflightAtHigh-m.InflightAtLow >= s.minInflightGap && + m.HighTTFT > m.LowTTFT && + m.LowTTFT > floor && + m.InflightAtLow > 0 cur := float64(m.Requests) - if cur < lowInflight && lowInflight > 0 { - // Below the low anchor, interpolate the low segment (0, floor) -> (lowInflight, lowTTFT) - // instead of extrapolating the steep queueing secant. The secant's slope was measured in the - // high-load region and, run backwards, dives below the floor within a few requests — pinning a - // draining-but-still-loaded pool at its idle latency. TTFT flattens toward the floor as the - // queue drains, so this segment tracks that. - eff = floor + cur*(lowTTFT-floor)/lowInflight - } else { - // At/above the low anchor: the in-cloud secant (lowInflight, lowTTFT) -> (InflightAtHigh, P50). - eff = lowTTFT + (cur-lowInflight)*(m.HighTTFT-lowTTFT)/(m.InflightAtHigh-lowInflight) - } - if eff < floor { - eff = floor // mathematically redundant now; kept as a defensive guard + switch { + case useLow && cur < m.InflightAtLow: + // (0, floor) -> low point. Extending the steeper high-load slope backwards instead would + // predict a draining-but-loaded pool at its idle latency. + pred = floor + cur*(m.LowTTFT-floor)/m.InflightAtLow + case useLow: + // low -> high point, extended beyond it. + pred = m.LowTTFT + (cur-m.InflightAtLow)*(m.HighTTFT-m.LowTTFT)/(m.InflightAtHigh-m.InflightAtLow) + default: + // Low point inadmissible: floor chord (0, floor) -> high point, extended beyond it. + pred = floor + cur*(m.HighTTFT-floor)/m.InflightAtHigh } - return eff, true, true + return max(pred, floor), true, true } // metricsFor reads the TTFT percentile metrics an extractor published for the model.