From 95ab800321dc74d227e0c4efcbe6637526ac606a Mon Sep 17 00:00:00 2001 From: mkoushni Date: Mon, 1 Jun 2026 17:57:12 +0300 Subject: [PATCH 1/6] feat(deploy): add Kustomize deployment option (#125) Signed-off-by: mkoushni --- Makefile | 26 +++ README.md | 30 +++ config/kustomize/README.md | 187 ++++++++++++++++++ config/kustomize/base/configmap.yaml | 20 ++ config/kustomize/base/deployment.yaml | 39 ++++ config/kustomize/base/kustomization.yaml | 9 + config/kustomize/base/rbac.yaml | 20 ++ config/kustomize/base/service.yaml | 13 ++ config/kustomize/base/serviceaccount.yaml | 4 + .../overlays/default/kustomization.yaml | 9 + .../overlays/gke/gcproutingextension.yaml | 31 +++ .../overlays/gke/healthcheckpolicy.yaml | 18 ++ .../kustomize/overlays/gke/kustomization.yaml | 11 ++ .../overlays/istio/destinationrule.yaml | 13 ++ .../kustomize/overlays/istio/envoyfilter.yaml | 39 ++++ .../overlays/istio/kustomization.yaml | 12 ++ pkg/config/loader/configloader_test.go | 8 +- pkg/config/loader/defaults.go | 9 +- test/integration/body_mutation_test.go | 8 +- test/integration/hermetic_test.go | 2 +- test/integration/util.go | 9 +- 21 files changed, 504 insertions(+), 13 deletions(-) create mode 100644 config/kustomize/README.md create mode 100644 config/kustomize/base/configmap.yaml create mode 100644 config/kustomize/base/deployment.yaml create mode 100644 config/kustomize/base/kustomization.yaml create mode 100644 config/kustomize/base/rbac.yaml create mode 100644 config/kustomize/base/service.yaml create mode 100644 config/kustomize/base/serviceaccount.yaml create mode 100644 config/kustomize/overlays/default/kustomization.yaml create mode 100644 config/kustomize/overlays/gke/gcproutingextension.yaml create mode 100644 config/kustomize/overlays/gke/healthcheckpolicy.yaml create mode 100644 config/kustomize/overlays/gke/kustomization.yaml create mode 100644 config/kustomize/overlays/istio/destinationrule.yaml create mode 100644 config/kustomize/overlays/istio/envoyfilter.yaml create mode 100644 config/kustomize/overlays/istio/kustomization.yaml diff --git a/Makefile b/Makefile index 88a5fc4f..173f4dad 100644 --- a/Makefile +++ b/Makefile @@ -31,6 +31,9 @@ KIND_CLUSTER_NAME ?= ipp-e2e # Tools GOLANGCI_LINT_VERSION ?= v2.8.0 +KUSTOMIZE ?= $(LOCALBIN)/kustomize +KUSTOMIZE_VERSION ?= v5.4.3 +KUSTOMIZE_OVERLAY ?= default .DEFAULT_GOAL := help @@ -128,6 +131,29 @@ $(YQ): | $(LOCALBIN) helm-push: yq helm-install ## Package and push the payload-processor Helm chart. CHART=$(CHART) EXTRA_TAG="$(EXTRA_TAG)" IMAGE_REPOSITORY="$(IMAGE_REPOSITORY)" YQ="$(YQ)" HELM="$(HELM)" ./hack/push-chart.sh +##@ Deployment + +.PHONY: kustomize +kustomize: ## Download kustomize locally if necessary + @test -f $(KUSTOMIZE) || { \ + mkdir -p $(LOCALBIN); \ + curl -sSLo /tmp/kustomize.tar.gz https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2F$(KUSTOMIZE_VERSION)/kustomize_$(KUSTOMIZE_VERSION)_linux_amd64.tar.gz && \ + tar -xzf /tmp/kustomize.tar.gz -C $(LOCALBIN) && \ + rm /tmp/kustomize.tar.gz; \ + } + +.PHONY: kustomize-build +kustomize-build: kustomize ## Render Kustomize manifests (KUSTOMIZE_OVERLAY=default|istio|gke) + $(KUSTOMIZE) build config/kustomize/overlays/$(KUSTOMIZE_OVERLAY) + +.PHONY: kustomize-deploy +kustomize-deploy: kustomize ## Deploy using Kustomize (KUSTOMIZE_OVERLAY=default|istio|gke) + $(KUSTOMIZE) build config/kustomize/overlays/$(KUSTOMIZE_OVERLAY) | kubectl apply -f - + +.PHONY: kustomize-undeploy +kustomize-undeploy: kustomize ## Remove Kustomize deployment (KUSTOMIZE_OVERLAY=default|istio|gke) + $(KUSTOMIZE) build config/kustomize/overlays/$(KUSTOMIZE_OVERLAY) | kubectl delete --ignore-not-found -f - + ##@ CI Helpers .PHONY: ci-lint diff --git a/README.md b/README.md index b6e74122..bd770acd 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,35 @@ Helm chart provisions the provider-specific integration automatically: - **GKE** — Installs a `GCPRoutingExtension` that registers IPP as a routing extension. - **None** — Deploys the core IPP resources (Deployment, Service, config, RBAC) but no proxy integration; you wire that up yourself. +## Deployment + +The payload processor can be deployed using either **Helm** or **Kustomize**. + +### Helm + +```bash +helm install payload-processor ./config/charts/payload-processor \ + --set provider.name=[gke|istio] \ + --set inferenceGateway.name=inference-gateway +``` + +See [config/charts/payload-processor/README.md](config/charts/payload-processor/README.md) for the full parameter reference. + +### Kustomize + +```bash +# No provider (Deployment + Service only) +kubectl kustomize config/kustomize/overlays/default | kubectl apply -f - + +# Istio (adds EnvoyFilter + DestinationRule) +kubectl kustomize config/kustomize/overlays/istio | kubectl apply -f - + +# GKE (adds GCPRoutingExtension + HealthCheckPolicy) +kubectl kustomize config/kustomize/overlays/gke | kubectl apply -f - +``` + +See [config/kustomize/README.md](config/kustomize/README.md) for customization options (namespace, image tag, custom config, multi-namespace RBAC). + ## Documentation | Document | Description | @@ -49,6 +78,7 @@ Helm chart provisions the provider-specific integration automatically: | [Creating a Plugin](docs/create_new_plugin.md) | Tutorial for writing and registering a custom plugin. | | [Metrics](docs/metrics.md) | Prometheus metrics exposed by IPP. | | [Helm Chart](config/charts/payload-processor/README.md) | Chart install reference and values table. | +| [Kustomize](config/kustomize/README.md) | Kustomize overlay reference and customization options. | | [ModelSelector Proposal](docs/proposals/043-model-selection-framework/README.md) | Design of the model-selection framework. | For end-to-end deployment, see the [llm-d] project documentation and guides. diff --git a/config/kustomize/README.md b/config/kustomize/README.md new file mode 100644 index 00000000..6cf5435c --- /dev/null +++ b/config/kustomize/README.md @@ -0,0 +1,187 @@ +# Kustomize Deployment + +This directory provides [Kustomize](https://kustomize.io/) manifests for deploying the +Inference Payload Processor (IPP). It mirrors the same resources as the Helm chart at +`config/charts/payload-processor/` and is the recommended path for integrations such as +[llm-d-benchmark](https://github.com/llm-d/llm-d-benchmark) and GitOps workflows. + +## Structure + +``` +config/kustomize/ +├── base/ # Core resources (provider-agnostic) +│ ├── kustomization.yaml +│ ├── deployment.yaml # Deployment +│ ├── service.yaml # ClusterIP Service on port 9004 (HTTP2) +│ ├── serviceaccount.yaml # ServiceAccount +│ ├── rbac.yaml # Role + RoleBinding (single-namespace) +│ └── configmap.yaml # Default PayloadProcessorConfig +└── overlays/ + ├── default/ # No provider — Deployment + Service only + ├── istio/ # Adds EnvoyFilter + DestinationRule + └── gke/ # Adds GCPRoutingExtension + HealthCheckPolicy +``` + +## Quick Start + +### Prerequisites + +- `kubectl` ≥ 1.24 +- `kustomize` ≥ 5.0 (or the `kustomize` embedded in `kubectl`) +- A running Kubernetes cluster with an Inference Gateway deployed + +### Deploy (no provider) + +```bash +# Render to stdout +kubectl kustomize config/kustomize/overlays/default + +# Apply directly +kubectl kustomize config/kustomize/overlays/default | kubectl apply -f - + +# Or via make +make kustomize-deploy +``` + +### Deploy with Istio + +```bash +kubectl kustomize config/kustomize/overlays/istio | kubectl apply -f - + +# Or via make +make kustomize-deploy KUSTOMIZE_OVERLAY=istio +``` + +### Deploy with GKE + +```bash +kubectl kustomize config/kustomize/overlays/gke | kubectl apply -f - + +# Or via make +make kustomize-deploy KUSTOMIZE_OVERLAY=gke +``` + +### Undeploy + +```bash +make kustomize-undeploy # default overlay +make kustomize-undeploy KUSTOMIZE_OVERLAY=istio +make kustomize-undeploy KUSTOMIZE_OVERLAY=gke +``` + +## Customization + +### Change the target namespace + +Edit the `namespace:` field in the overlay's `kustomization.yaml`: + +```yaml +# config/kustomize/overlays/default/kustomization.yaml +namespace: my-namespace # ← change this +``` + +Or patch it inline from the command line: + +```bash +cd config/kustomize/overlays/default +kustomize edit set namespace my-namespace +``` + +> **Istio and GKE users:** The `cluster_name` in `overlays/istio/envoyfilter.yaml` and the +> `host` in `overlays/istio/destinationrule.yaml` embed the namespace as part of the FQDN +> (`payload-processor..svc.cluster.local`). Update those fields to match your +> chosen namespace. + +### Change the container image + +Add an `images` override in your overlay's `kustomization.yaml`: + +```yaml +images: + - name: ghcr.io/llm-d/llm-d-inference-payload-processor + newTag: v0.3.0 +``` + +### Change the Gateway name + +Patch the `targetRefs[0].name` field in `envoyfilter.yaml` (Istio) or +`gcproutingextension.yaml` (GKE) using a strategic merge patch: + +```yaml +# overlays/istio/gateway-patch.yaml +apiVersion: networking.istio.io/v1alpha3 +kind: EnvoyFilter +metadata: + name: payload-processor +spec: + targetRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: my-custom-gateway # ← your Gateway name +``` + +```yaml +# overlays/istio/kustomization.yaml (add to existing file) +patches: + - path: gateway-patch.yaml +``` + +### Use a custom IPP config + +Add a `configMapGenerator` entry in your overlay's `kustomization.yaml` to merge your own +`PayloadProcessorConfig`: + +```yaml +configMapGenerator: + - name: payload-processor + behavior: merge + files: + - custom-ipp-config.yaml=path/to/your/config.yaml +``` + +Then update the `--config-file` arg in a Deployment patch to point to +`/config/custom-ipp-config.yaml`. + +### Multi-namespace RBAC + +The base uses a namespace-scoped `Role`/`RoleBinding`. To watch ConfigMaps across +namespaces, create an overlay that replaces them with a `ClusterRole`/`ClusterRoleBinding`: + +```yaml +# overlays/multi-namespace/kustomization.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: my-namespace + +resources: + - ../../base + - clusterrole.yaml + - clusterrolebinding.yaml + +patches: + - target: + kind: Role + patch: |- + $patch: delete + apiVersion: rbac.authorization.k8s.io/v1 + kind: Role + metadata: + name: payload-processor-configmap-reader + - target: + kind: RoleBinding + patch: |- + $patch: delete + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: payload-processor-configmap-reader +``` + +## Notes + +- This chart should only be deployed once per Gateway (same constraint as the Helm chart). +- The `base/` layer intentionally omits `metadata.namespace` so that the overlay's + `namespace:` field is the single source of truth. +- For production use, pin the image tag and consider setting resource requests/limits via + a Deployment patch. diff --git a/config/kustomize/base/configmap.yaml b/config/kustomize/base/configmap.yaml new file mode 100644 index 00000000..d4c836b2 --- /dev/null +++ b/config/kustomize/base/configmap.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: payload-processor +data: + default-ipp-config.yaml: | + apiVersion: llm-d.ai/v1alpha1 + kind: PayloadProcessorConfig + plugins: + - type: body-field-to-header + parameters: + fieldName: model + headerName: X-Gateway-Model-Name + - type: base-model-to-header + profiles: + - name: default + plugins: + request: + - pluginRef: body-field-to-header + - pluginRef: base-model-to-header diff --git a/config/kustomize/base/deployment.yaml b/config/kustomize/base/deployment.yaml new file mode 100644 index 00000000..b6a69594 --- /dev/null +++ b/config/kustomize/base/deployment.yaml @@ -0,0 +1,39 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: payload-processor +spec: + replicas: 1 + selector: + matchLabels: + app: payload-processor + template: + metadata: + labels: + app: payload-processor + spec: + serviceAccountName: payload-processor + containers: + - name: payload-processor + image: ghcr.io/llm-d/llm-d-inference-payload-processor:main + imagePullPolicy: IfNotPresent + args: + - --config-file + - /config/default-ipp-config.yaml + - --v=3 + - --tracing=false + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + ports: + - containerPort: 9004 + - containerPort: 9005 + volumeMounts: + - name: config-volume + mountPath: /config + volumes: + - name: config-volume + configMap: + name: payload-processor diff --git a/config/kustomize/base/kustomization.yaml b/config/kustomize/base/kustomization.yaml new file mode 100644 index 00000000..8b75a993 --- /dev/null +++ b/config/kustomize/base/kustomization.yaml @@ -0,0 +1,9 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - configmap.yaml + - serviceaccount.yaml + - rbac.yaml + - deployment.yaml + - service.yaml diff --git a/config/kustomize/base/rbac.yaml b/config/kustomize/base/rbac.yaml new file mode 100644 index 00000000..1afb4a6d --- /dev/null +++ b/config/kustomize/base/rbac.yaml @@ -0,0 +1,20 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: payload-processor-configmap-reader +rules: + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: payload-processor-configmap-reader +subjects: + - kind: ServiceAccount + name: payload-processor +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: payload-processor-configmap-reader diff --git a/config/kustomize/base/service.yaml b/config/kustomize/base/service.yaml new file mode 100644 index 00000000..9ecdd049 --- /dev/null +++ b/config/kustomize/base/service.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Service +metadata: + name: payload-processor +spec: + selector: + app: payload-processor + ports: + - protocol: TCP + port: 9004 + targetPort: 9004 + appProtocol: HTTP2 + type: ClusterIP diff --git a/config/kustomize/base/serviceaccount.yaml b/config/kustomize/base/serviceaccount.yaml new file mode 100644 index 00000000..8af6ace9 --- /dev/null +++ b/config/kustomize/base/serviceaccount.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: payload-processor diff --git a/config/kustomize/overlays/default/kustomization.yaml b/config/kustomize/overlays/default/kustomization.yaml new file mode 100644 index 00000000..75363fd2 --- /dev/null +++ b/config/kustomize/overlays/default/kustomization.yaml @@ -0,0 +1,9 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# Override the namespace for all resources in the base. +# Change this to the namespace where you want to deploy the payload processor. +namespace: default + +resources: + - ../../base diff --git a/config/kustomize/overlays/gke/gcproutingextension.yaml b/config/kustomize/overlays/gke/gcproutingextension.yaml new file mode 100644 index 00000000..dc5faafb --- /dev/null +++ b/config/kustomize/overlays/gke/gcproutingextension.yaml @@ -0,0 +1,31 @@ +apiVersion: networking.gke.io/v1 +kind: GCPRoutingExtension +metadata: + # Namespace is set by kustomization.yaml + name: payload-processor +spec: + targetRefs: + - group: gateway.networking.k8s.io + kind: Gateway + # Update to match your Gateway name. + name: inference-gateway + extensionChains: + - name: chain1 + extensions: + - name: ext1 + authority: "myext.com" + timeout: 1s + supportedEvents: + - RequestHeaders + - RequestBody + - RequestTrailers + - ResponseHeaders + - ResponseBody + - ResponseTrailers + requestBodySendMode: "FullDuplexStreamed" + responseBodySendMode: "FullDuplexStreamed" + backendRef: + group: "" + kind: Service + name: payload-processor + port: 9004 diff --git a/config/kustomize/overlays/gke/healthcheckpolicy.yaml b/config/kustomize/overlays/gke/healthcheckpolicy.yaml new file mode 100644 index 00000000..13d33f34 --- /dev/null +++ b/config/kustomize/overlays/gke/healthcheckpolicy.yaml @@ -0,0 +1,18 @@ +apiVersion: networking.gke.io/v1 +kind: HealthCheckPolicy +metadata: + # Namespace is set by kustomization.yaml + name: payload-processor-healthcheck +spec: + default: + logConfig: + enabled: true + config: + type: "GRPC" + grpcHealthCheck: + portSpecification: "USE_FIXED_PORT" + port: 9005 + targetRef: + group: "" + kind: Service + name: payload-processor diff --git a/config/kustomize/overlays/gke/kustomization.yaml b/config/kustomize/overlays/gke/kustomization.yaml new file mode 100644 index 00000000..a7a49f3b --- /dev/null +++ b/config/kustomize/overlays/gke/kustomization.yaml @@ -0,0 +1,11 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# Override the namespace for all resources in the base. +# Change this to the namespace where you want to deploy the payload processor. +namespace: default + +resources: + - ../../base + - gcproutingextension.yaml + - healthcheckpolicy.yaml diff --git a/config/kustomize/overlays/istio/destinationrule.yaml b/config/kustomize/overlays/istio/destinationrule.yaml new file mode 100644 index 00000000..84ea7705 --- /dev/null +++ b/config/kustomize/overlays/istio/destinationrule.yaml @@ -0,0 +1,13 @@ +apiVersion: networking.istio.io/v1 +kind: DestinationRule +metadata: + # Namespace is set by kustomization.yaml + name: payload-processor +spec: + # Update the namespace segment (currently "default") to match + # the namespace set in kustomization.yaml. + host: payload-processor.default.svc.cluster.local + trafficPolicy: + tls: + mode: SIMPLE + insecureSkipVerify: true diff --git a/config/kustomize/overlays/istio/envoyfilter.yaml b/config/kustomize/overlays/istio/envoyfilter.yaml new file mode 100644 index 00000000..a1ce7db4 --- /dev/null +++ b/config/kustomize/overlays/istio/envoyfilter.yaml @@ -0,0 +1,39 @@ +apiVersion: networking.istio.io/v1alpha3 +kind: EnvoyFilter +metadata: + # Namespace is set by kustomization.yaml + name: payload-processor +spec: + targetRefs: + - group: gateway.networking.k8s.io + kind: Gateway + # Update to match your Gateway name. + name: inference-gateway + configPatches: + - applyTo: HTTP_FILTER + match: + context: GATEWAY + listener: + filterChain: + filter: + name: "envoy.filters.network.http_connection_manager" + patch: + operation: INSERT_FIRST + value: + name: envoy.filters.http.ext_proc.payload-processor + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.ext_proc.v3.ExternalProcessor + failure_mode_allow: false + allow_mode_override: true + processing_mode: + request_header_mode: "SEND" + response_header_mode: "SEND" + request_body_mode: "FULL_DUPLEX_STREAMED" + response_body_mode: "FULL_DUPLEX_STREAMED" + request_trailer_mode: "SEND" + response_trailer_mode: "SEND" + grpc_service: + envoy_grpc: + # Update the namespace segment (currently "default") to match + # the namespace set in kustomization.yaml. + cluster_name: outbound|9004||payload-processor.default.svc.cluster.local diff --git a/config/kustomize/overlays/istio/kustomization.yaml b/config/kustomize/overlays/istio/kustomization.yaml new file mode 100644 index 00000000..64272613 --- /dev/null +++ b/config/kustomize/overlays/istio/kustomization.yaml @@ -0,0 +1,12 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# Override the namespace for all resources in the base. +# IMPORTANT: Also update the cluster_name in envoyfilter.yaml and +# the host in destinationrule.yaml to match this namespace. +namespace: default + +resources: + - ../../base + - envoyfilter.yaml + - destinationrule.yaml diff --git a/pkg/config/loader/configloader_test.go b/pkg/config/loader/configloader_test.go index c8041201..0ac52573 100644 --- a/pkg/config/loader/configloader_test.go +++ b/pkg/config/loader/configloader_test.go @@ -73,8 +73,8 @@ func TestLoadRawConfiguration(t *testing.T) { configText: successConfigText, want: &configapi.PayloadProcessorConfig{ TypeMeta: metav1.TypeMeta{ - Kind: "PayloadProcessorConfig", - APIVersion: "llm-d.ai/v1alpha1", + Kind: configKind, + APIVersion: configAPIVersion, }, Plugins: []configapi.PluginSpec{ {Name: testRequestProcType, Type: testRequestProcType}, @@ -91,8 +91,8 @@ func TestLoadRawConfiguration(t *testing.T) { configText: "", want: &configapi.PayloadProcessorConfig{ TypeMeta: metav1.TypeMeta{ - APIVersion: "llm-d.ai/v1alpha1", - Kind: "PayloadProcessorConfig", + APIVersion: configAPIVersion, + Kind: configKind, }, Plugins: []configapi.PluginSpec{ { diff --git a/pkg/config/loader/defaults.go b/pkg/config/loader/defaults.go index 15d1a30a..bec14736 100644 --- a/pkg/config/loader/defaults.go +++ b/pkg/config/loader/defaults.go @@ -31,11 +31,16 @@ import ( "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/plugins/requesthandling/profilepicker/single" ) +const ( + configAPIVersion = "llm-d.ai/v1alpha1" + configKind = "PayloadProcessorConfig" +) + func loadDefaultConfig() *configapi.PayloadProcessorConfig { return &configapi.PayloadProcessorConfig{ TypeMeta: metav1.TypeMeta{ - APIVersion: "llm-d.ai/v1alpha1", - Kind: "PayloadProcessorConfig", + APIVersion: configAPIVersion, + Kind: configKind, }, Plugins: []configapi.PluginSpec{ { diff --git a/test/integration/body_mutation_test.go b/test/integration/body_mutation_test.go index 8d088aef..0146b5e5 100644 --- a/test/integration/body_mutation_test.go +++ b/test/integration/body_mutation_test.go @@ -62,7 +62,7 @@ func TestBodyMutation(t *testing.T) { baseModelToHeaderPlugin := &basemodelextractor.BaseModelToHeaderPlugin{AdaptersStore: basemodelextractor.NewAdaptersStore()} h := NewHarnessWithPlugins(t, ctx, []requesthandling.RequestProcessor{plugin, baseModelToHeaderPlugin}, []requesthandling.ResponseProcessor{}) - body := map[string]any{"prompt": "hello"} + body := map[string]any{bodyFieldPrompt: "hello"} bodyBytes, _ := json.Marshal(body) reqs := []*extProcPb.ProcessingRequest{ @@ -88,8 +88,8 @@ func TestBodyMutation(t *testing.T) { } wantBody, _ := json.Marshal(map[string]any{ - "prompt": "hello", - "injected": "test-value", + bodyFieldPrompt: "hello", + "injected": "test-value", }) wantResponses := []*extProcPb.ProcessingResponse{ { @@ -101,7 +101,7 @@ func TestBodyMutation(t *testing.T) { SetHeaders: []*envoyCorev3.HeaderValueOption{ { Header: &envoyCorev3.HeaderValue{ - Key: "Content-Length", + Key: headerContentLength, RawValue: []byte(strconv.Itoa(len(wantBody))), }, AppendAction: envoyCorev3.HeaderValueOption_OVERWRITE_IF_EXISTS_OR_ADD, diff --git a/test/integration/hermetic_test.go b/test/integration/hermetic_test.go index 4cf90726..628da313 100644 --- a/test/integration/hermetic_test.go +++ b/test/integration/hermetic_test.go @@ -79,7 +79,7 @@ func TestBodyBasedRouting(t *testing.T) { SetHeaders: []*envoyCorev3.HeaderValueOption{ { Header: &envoyCorev3.HeaderValue{ - Key: "Content-Length", + Key: headerContentLength, RawValue: []byte("50"), }, AppendAction: envoyCorev3.HeaderValueOption_OVERWRITE_IF_EXISTS_OR_ADD, diff --git a/test/integration/util.go b/test/integration/util.go index 008065b0..916fad38 100644 --- a/test/integration/util.go +++ b/test/integration/util.go @@ -23,6 +23,11 @@ import ( extProcPb "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" ) +const ( + headerContentLength = "Content-Length" + bodyFieldPrompt = "prompt" +) + // --- Response Expectations (Streaming) --- // ExpectHeader asserts that the payload processor set the specific model header and cleared the route cache. @@ -37,7 +42,7 @@ func ExpectHeader(modelName, baseModelName string, contentLength string) *extPro SetHeaders: []*envoyCorev3.HeaderValueOption{ { Header: &envoyCorev3.HeaderValue{ - Key: "Content-Length", + Key: headerContentLength, RawValue: []byte(contentLength), }, AppendAction: envoyCorev3.HeaderValueOption_OVERWRITE_IF_EXISTS_OR_ADD, @@ -68,7 +73,7 @@ func ExpectHeader(modelName, baseModelName string, contentLength string) *extPro // The payload processor buffers the body to inspect it, then sends it downstream as a single chunk (usually). func ExpectBodyPassThrough(prompt, model string) *extProcPb.ProcessingResponse { j := map[string]any{ - "max_tokens": 100, "prompt": prompt, "temperature": 0, + "max_tokens": 100, bodyFieldPrompt: prompt, "temperature": 0, } if model != "" { j["model"] = model From 46c94362f51d250af22bb4e575524c1c4eb03543 Mon Sep 17 00:00:00 2001 From: mkoushni Date: Tue, 14 Jul 2026 14:24:22 +0300 Subject: [PATCH 2/6] ci: bump golangci-lint-action to v9 to fix Node 20 deprecation warning v9 moves the action's runtime from Node 20 to Node 24, per the upstream release notes. Signed-off-by: mkoushni --- .github/workflows/ci-pr-checks.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-pr-checks.yaml b/.github/workflows/ci-pr-checks.yaml index d9730a8a..1d8cd675 100644 --- a/.github/workflows/ci-pr-checks.yaml +++ b/.github/workflows/ci-pr-checks.yaml @@ -50,7 +50,7 @@ jobs: run: go mod download - name: Run golangci-lint - uses: golangci/golangci-lint-action@v8 + uses: golangci/golangci-lint-action@v9 with: version: v2.8.0 args: "" From af5b45a7c0ba16f0073062d03792290753300a89 Mon Sep 17 00:00:00 2001 From: mkoushni Date: Tue, 14 Jul 2026 14:37:04 +0300 Subject: [PATCH 3/6] test: fix gofmt formatting and remove unused test helper golangci-lint flagged plugin_test.go for improper gofmt formatting and an unused waitForTokenPrices helper with no call sites. Signed-off-by: mkoushni --- .../plugins/datalayer/requestcostmetadata/plugin_test.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/pkg/framework/plugins/datalayer/requestcostmetadata/plugin_test.go b/pkg/framework/plugins/datalayer/requestcostmetadata/plugin_test.go index 2b1e52e9..ea02ba29 100644 --- a/pkg/framework/plugins/datalayer/requestcostmetadata/plugin_test.go +++ b/pkg/framework/plugins/datalayer/requestcostmetadata/plugin_test.go @@ -148,7 +148,6 @@ func newTestExtractor(t *testing.T) (*RequestCostMetadataExtractor, datalayer.Da // --- Factory tests --- - func TestExtractorFactory_HonorsConfig(t *testing.T) { ds := datastore.NewFakeDataStore() raw := json.RawMessage(`{"compression":50,"flushIntervalDuration":"1m"}`) @@ -360,9 +359,9 @@ func TestExtract_MultipleModels(t *testing.T) { setTokenPrices(ds, "m2", 5e-7, 1e-6) // input $0.5/M, output $1/M // Batch with interleaved models: m1, m2, m1 - ev1 := makeResponseEvent("m1", 100, 50, false) // cost = 100*1e-6 + 50*2e-6 = 1e-4 + 1e-4 = 2e-4 - ev2 := makeResponseEvent("m2", 200, 100, false) // cost = 200*5e-7 + 100*1e-6 = 1e-4 + 1e-4 = 2e-4 - ev3 := makeResponseEvent("m1", 50, 100, false) // cost = 50*1e-6 + 100*2e-6 = 5e-5 + 2e-4 = 2.5e-4 + ev1 := makeResponseEvent("m1", 100, 50, false) // cost = 100*1e-6 + 50*2e-6 = 1e-4 + 1e-4 = 2e-4 + ev2 := makeResponseEvent("m2", 200, 100, false) // cost = 200*5e-7 + 100*1e-6 = 1e-4 + 1e-4 = 2e-4 + ev3 := makeResponseEvent("m1", 50, 100, false) // cost = 50*1e-6 + 100*2e-6 = 5e-5 + 2e-4 = 2.5e-4 if err := ext.Extract(context.Background(), []dlsrc.Event{ev1, ev2, ev3}); err != nil { t.Fatalf("Extract: %v", err) @@ -622,4 +621,3 @@ func TestExtract_FlushIntervalGating(t *testing.T) { t.Errorf("expected published digest to have samples, got count=0") } } - From 084e50fb9c6a77ae9999507c2d96934586596df0 Mon Sep 17 00:00:00 2001 From: mkoushni Date: Sun, 19 Jul 2026 13:32:51 +0300 Subject: [PATCH 4/6] fix(makefile): install kustomize via go install for cross-platform support The kustomize target hardcoded a linux_amd64 release tarball download, which broke on macOS and ARM dev environments. Use the existing go-install-tool helper (already used for helm, yq, controller-gen) so installation works on any OS/arch supported by the Go toolchain. Signed-off-by: mkoushni --- Makefile | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 173f4dad..39964aa0 100644 --- a/Makefile +++ b/Makefile @@ -134,13 +134,9 @@ helm-push: yq helm-install ## Package and push the payload-processor Helm chart. ##@ Deployment .PHONY: kustomize -kustomize: ## Download kustomize locally if necessary - @test -f $(KUSTOMIZE) || { \ - mkdir -p $(LOCALBIN); \ - curl -sSLo /tmp/kustomize.tar.gz https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2F$(KUSTOMIZE_VERSION)/kustomize_$(KUSTOMIZE_VERSION)_linux_amd64.tar.gz && \ - tar -xzf /tmp/kustomize.tar.gz -C $(LOCALBIN) && \ - rm /tmp/kustomize.tar.gz; \ - } +kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary. +$(KUSTOMIZE): | $(LOCALBIN) + $(call go-install-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v5,$(KUSTOMIZE_VERSION)) .PHONY: kustomize-build kustomize-build: kustomize ## Render Kustomize manifests (KUSTOMIZE_OVERLAY=default|istio|gke) From 96614fbbb29dcfb15821bfdbe2fd178bea4d763c Mon Sep 17 00:00:00 2001 From: mkoushni Date: Sun, 19 Jul 2026 13:51:19 +0300 Subject: [PATCH 5/6] fix(kustomize): eliminate namespace drift in Istio overlay via replacements The EnvoyFilter cluster_name and DestinationRule host embedded the namespace as a hardcoded FQDN segment, requiring manual updates that could silently drift out of sync with the namespace: field in kustomization.yaml. Use Kustomize replacements to derive the namespace from the Service resource and inject it into both fields at build time, so kustomization.yaml's namespace: is the single source of truth. Signed-off-by: mkoushni --- config/kustomize/README.md | 10 +++--- .../overlays/istio/destinationrule.yaml | 5 +-- .../kustomize/overlays/istio/envoyfilter.yaml | 5 +-- .../overlays/istio/kustomization.yaml | 31 +++++++++++++++++-- 4 files changed, 41 insertions(+), 10 deletions(-) diff --git a/config/kustomize/README.md b/config/kustomize/README.md index 6cf5435c..aad2ffa5 100644 --- a/config/kustomize/README.md +++ b/config/kustomize/README.md @@ -87,10 +87,12 @@ cd config/kustomize/overlays/default kustomize edit set namespace my-namespace ``` -> **Istio and GKE users:** The `cluster_name` in `overlays/istio/envoyfilter.yaml` and the -> `host` in `overlays/istio/destinationrule.yaml` embed the namespace as part of the FQDN -> (`payload-processor..svc.cluster.local`). Update those fields to match your -> chosen namespace. +> **Istio users:** The `cluster_name` in `overlays/istio/envoyfilter.yaml` and the `host` in +> `overlays/istio/destinationrule.yaml` embed the namespace as part of the FQDN +> (`payload-processor..svc.cluster.local`). This is handled automatically — the +> `replacements` block in `overlays/istio/kustomization.yaml` injects the overlay's +> `namespace:` value into both fields at build time, so there is nothing to edit manually and +> no risk of drift between them. ### Change the container image diff --git a/config/kustomize/overlays/istio/destinationrule.yaml b/config/kustomize/overlays/istio/destinationrule.yaml index 84ea7705..e708ee4d 100644 --- a/config/kustomize/overlays/istio/destinationrule.yaml +++ b/config/kustomize/overlays/istio/destinationrule.yaml @@ -4,8 +4,9 @@ metadata: # Namespace is set by kustomization.yaml name: payload-processor spec: - # Update the namespace segment (currently "default") to match - # the namespace set in kustomization.yaml. + # The namespace segment below is a placeholder; it is replaced at build + # time by the `replacements` block in kustomization.yaml to match the + # overlay's `namespace:`. host: payload-processor.default.svc.cluster.local trafficPolicy: tls: diff --git a/config/kustomize/overlays/istio/envoyfilter.yaml b/config/kustomize/overlays/istio/envoyfilter.yaml index a1ce7db4..338423d2 100644 --- a/config/kustomize/overlays/istio/envoyfilter.yaml +++ b/config/kustomize/overlays/istio/envoyfilter.yaml @@ -34,6 +34,7 @@ spec: response_trailer_mode: "SEND" grpc_service: envoy_grpc: - # Update the namespace segment (currently "default") to match - # the namespace set in kustomization.yaml. + # The namespace segment below is a placeholder; it is + # replaced at build time by the `replacements` block in + # kustomization.yaml to match the overlay's `namespace:`. cluster_name: outbound|9004||payload-processor.default.svc.cluster.local diff --git a/config/kustomize/overlays/istio/kustomization.yaml b/config/kustomize/overlays/istio/kustomization.yaml index 64272613..2fe78252 100644 --- a/config/kustomize/overlays/istio/kustomization.yaml +++ b/config/kustomize/overlays/istio/kustomization.yaml @@ -2,11 +2,38 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization # Override the namespace for all resources in the base. -# IMPORTANT: Also update the cluster_name in envoyfilter.yaml and -# the host in destinationrule.yaml to match this namespace. +# The namespace segment embedded in the EnvoyFilter cluster_name and the +# DestinationRule host (below, via `replacements`) is derived from this +# field automatically, so it never drifts out of sync. namespace: default resources: - ../../base - envoyfilter.yaml - destinationrule.yaml + +# Inject the overlay namespace into the FQDN segments of the EnvoyFilter's +# cluster_name and the DestinationRule's host, so changing `namespace:` +# above is the single source of truth and can't silently drift. +replacements: + - source: + kind: Service + name: payload-processor + fieldPath: metadata.namespace + targets: + - select: + kind: EnvoyFilter + name: payload-processor + fieldPaths: + - spec.configPatches.0.patch.value.typed_config.grpc_service.envoy_grpc.cluster_name + options: + delimiter: "." + index: 1 + - select: + kind: DestinationRule + name: payload-processor + fieldPaths: + - spec.host + options: + delimiter: "." + index: 1 From 221f3fbe569f5402dd3fe6bc1704b731f5e855d2 Mon Sep 17 00:00:00 2001 From: mkoushni Date: Sun, 19 Jul 2026 14:12:24 +0300 Subject: [PATCH 6/6] fix(kustomize): address production-readiness review feedback - Add resource requests/limits to the base Deployment so the Pod can't be OOMKilled or starve neighbors in a shared cluster. - Add readiness/liveness probes using the gRPC health protocol served on port 9005 (grpc.health.v1, see cmd/runner/health.go) so rolling updates and self-healing work correctly. - Name the Deployment's container ports and the Service port so Istio (and other mesh sidecars) detect the protocol instead of treating traffic as opaque TCP. - Document why insecureSkipVerify is set on the Istio DestinationRule (IPP serves a self-signed cert with no shared CA) so it isn't mistaken for an oversight or copied onto external-facing hosts. - Raise the GKE GCPRoutingExtension timeout from 1s to 10s, since IPP streams full request/response bodies and runs model-selection/cost logic that can exceed 1s under load. Signed-off-by: mkoushni --- config/kustomize/base/deployment.yaml | 27 +++++++++++++++++-- config/kustomize/base/service.yaml | 5 +++- .../overlays/gke/gcproutingextension.yaml | 7 ++++- .../overlays/istio/destinationrule.yaml | 5 ++++ 4 files changed, 40 insertions(+), 4 deletions(-) diff --git a/config/kustomize/base/deployment.yaml b/config/kustomize/base/deployment.yaml index b6a69594..71312f55 100644 --- a/config/kustomize/base/deployment.yaml +++ b/config/kustomize/base/deployment.yaml @@ -28,8 +28,31 @@ spec: fieldRef: fieldPath: metadata.namespace ports: - - containerPort: 9004 - - containerPort: 9005 + - name: grpc + containerPort: 9004 + - name: grpc-health + containerPort: 9005 + # Conservative starting point for a request/response processing + # sidecar; tune based on observed load (payload size, RPS) before + # running in production. + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + memory: 512Mi + # Port 9005 serves the standard gRPC health protocol + # (grpc.health.v1), see cmd/runner/health.go. + readinessProbe: + grpc: + port: 9005 + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + grpc: + port: 9005 + initialDelaySeconds: 15 + periodSeconds: 20 volumeMounts: - name: config-volume mountPath: /config diff --git a/config/kustomize/base/service.yaml b/config/kustomize/base/service.yaml index 9ecdd049..972c0b75 100644 --- a/config/kustomize/base/service.yaml +++ b/config/kustomize/base/service.yaml @@ -6,7 +6,10 @@ spec: selector: app: payload-processor ports: - - protocol: TCP + # Named so Istio (and other mesh sidecars) detect the protocol from the + # port name instead of treating the traffic as opaque TCP. + - name: grpc-ext-proc + protocol: TCP port: 9004 targetPort: 9004 appProtocol: HTTP2 diff --git a/config/kustomize/overlays/gke/gcproutingextension.yaml b/config/kustomize/overlays/gke/gcproutingextension.yaml index dc5faafb..764a5f6b 100644 --- a/config/kustomize/overlays/gke/gcproutingextension.yaml +++ b/config/kustomize/overlays/gke/gcproutingextension.yaml @@ -14,7 +14,12 @@ spec: extensions: - name: ext1 authority: "myext.com" - timeout: 1s + # IPP processes full request/response bodies (FullDuplexStreamed + # below) plus model-selection and cost-computation logic, so a + # 1s timeout is too aggressive under load or with larger payloads. + # 10s is a safer starting point; tune based on observed p99 + # request latency and payload size. + timeout: 10s supportedEvents: - RequestHeaders - RequestBody diff --git a/config/kustomize/overlays/istio/destinationrule.yaml b/config/kustomize/overlays/istio/destinationrule.yaml index e708ee4d..eb6dc7f5 100644 --- a/config/kustomize/overlays/istio/destinationrule.yaml +++ b/config/kustomize/overlays/istio/destinationrule.yaml @@ -11,4 +11,9 @@ spec: trafficPolicy: tls: mode: SIMPLE + # IPP serves TLS with a self-signed certificate generated at startup + # (see internal/tls/tls.go) rather than one issued by a shared/trusted + # CA, so Istio cannot validate it against a CA bundle. This setting + # is intentional for intra-cluster traffic to the sidecar; do not set + # this on a DestinationRule that targets an externally-facing host. insecureSkipVerify: true