diff --git a/.github/workflows/ci-pr-checks.yaml b/.github/workflows/ci-pr-checks.yaml index c92a37a2..f7146bda 100644 --- a/.github/workflows/ci-pr-checks.yaml +++ b/.github/workflows/ci-pr-checks.yaml @@ -50,7 +50,7 @@ jobs: run: go mod download - name: Run golangci-lint - uses: golangci/golangci-lint-action@v8 + uses: golangci/golangci-lint-action@v9 with: version: v2.8.0 args: "" diff --git a/Makefile b/Makefile index 88a5fc4f..39964aa0 100644 --- a/Makefile +++ b/Makefile @@ -31,6 +31,9 @@ KIND_CLUSTER_NAME ?= ipp-e2e # Tools GOLANGCI_LINT_VERSION ?= v2.8.0 +KUSTOMIZE ?= $(LOCALBIN)/kustomize +KUSTOMIZE_VERSION ?= v5.4.3 +KUSTOMIZE_OVERLAY ?= default .DEFAULT_GOAL := help @@ -128,6 +131,25 @@ $(YQ): | $(LOCALBIN) helm-push: yq helm-install ## Package and push the payload-processor Helm chart. CHART=$(CHART) EXTRA_TAG="$(EXTRA_TAG)" IMAGE_REPOSITORY="$(IMAGE_REPOSITORY)" YQ="$(YQ)" HELM="$(HELM)" ./hack/push-chart.sh +##@ Deployment + +.PHONY: kustomize +kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary. +$(KUSTOMIZE): | $(LOCALBIN) + $(call go-install-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v5,$(KUSTOMIZE_VERSION)) + +.PHONY: kustomize-build +kustomize-build: kustomize ## Render Kustomize manifests (KUSTOMIZE_OVERLAY=default|istio|gke) + $(KUSTOMIZE) build config/kustomize/overlays/$(KUSTOMIZE_OVERLAY) + +.PHONY: kustomize-deploy +kustomize-deploy: kustomize ## Deploy using Kustomize (KUSTOMIZE_OVERLAY=default|istio|gke) + $(KUSTOMIZE) build config/kustomize/overlays/$(KUSTOMIZE_OVERLAY) | kubectl apply -f - + +.PHONY: kustomize-undeploy +kustomize-undeploy: kustomize ## Remove Kustomize deployment (KUSTOMIZE_OVERLAY=default|istio|gke) + $(KUSTOMIZE) build config/kustomize/overlays/$(KUSTOMIZE_OVERLAY) | kubectl delete --ignore-not-found -f - + ##@ CI Helpers .PHONY: ci-lint diff --git a/README.md b/README.md index 78d25aa9..c19b6002 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,35 @@ Helm chart provisions the provider-specific integration automatically: - **GKE** — Installs a `GCPRoutingExtension` that registers IPP as a routing extension. - **None** — Deploys the core IPP resources (Deployment, Service, config, RBAC) but no proxy integration; you wire that up yourself. +## Deployment + +The payload processor can be deployed using either **Helm** or **Kustomize**. + +### Helm + +```bash +helm install payload-processor ./config/charts/payload-processor \ + --set provider.name=[gke|istio] \ + --set inferenceGateway.name=inference-gateway +``` + +See [config/charts/payload-processor/README.md](config/charts/payload-processor/README.md) for the full parameter reference. + +### Kustomize + +```bash +# No provider (Deployment + Service only) +kubectl kustomize config/kustomize/overlays/default | kubectl apply -f - + +# Istio (adds EnvoyFilter + DestinationRule) +kubectl kustomize config/kustomize/overlays/istio | kubectl apply -f - + +# GKE (adds GCPRoutingExtension + HealthCheckPolicy) +kubectl kustomize config/kustomize/overlays/gke | kubectl apply -f - +``` + +See [config/kustomize/README.md](config/kustomize/README.md) for customization options (namespace, image tag, custom config, multi-namespace RBAC). + ## Documentation | Document | Description | @@ -50,6 +79,7 @@ Helm chart provisions the provider-specific integration automatically: | [Creating a Plugin](docs/create_new_plugin.md) | Tutorial for writing and registering a custom plugin. | | [Metrics](docs/metrics.md) | Prometheus metrics exposed by IPP. | | [Helm Chart](config/charts/payload-processor/README.md) | Chart install reference and values table. | +| [Kustomize](config/kustomize/README.md) | Kustomize overlay reference and customization options. | | [ModelSelector Proposal](docs/proposals/043-model-selection-framework/README.md) | Design of the model-selection framework. | For end-to-end deployment, see the [llm-d] project documentation and guides. diff --git a/config/kustomize/README.md b/config/kustomize/README.md new file mode 100644 index 00000000..aad2ffa5 --- /dev/null +++ b/config/kustomize/README.md @@ -0,0 +1,189 @@ +# Kustomize Deployment + +This directory provides [Kustomize](https://kustomize.io/) manifests for deploying the +Inference Payload Processor (IPP). It mirrors the same resources as the Helm chart at +`config/charts/payload-processor/` and is the recommended path for integrations such as +[llm-d-benchmark](https://github.com/llm-d/llm-d-benchmark) and GitOps workflows. + +## Structure + +``` +config/kustomize/ +├── base/ # Core resources (provider-agnostic) +│ ├── kustomization.yaml +│ ├── deployment.yaml # Deployment +│ ├── service.yaml # ClusterIP Service on port 9004 (HTTP2) +│ ├── serviceaccount.yaml # ServiceAccount +│ ├── rbac.yaml # Role + RoleBinding (single-namespace) +│ └── configmap.yaml # Default PayloadProcessorConfig +└── overlays/ + ├── default/ # No provider — Deployment + Service only + ├── istio/ # Adds EnvoyFilter + DestinationRule + └── gke/ # Adds GCPRoutingExtension + HealthCheckPolicy +``` + +## Quick Start + +### Prerequisites + +- `kubectl` ≥ 1.24 +- `kustomize` ≥ 5.0 (or the `kustomize` embedded in `kubectl`) +- A running Kubernetes cluster with an Inference Gateway deployed + +### Deploy (no provider) + +```bash +# Render to stdout +kubectl kustomize config/kustomize/overlays/default + +# Apply directly +kubectl kustomize config/kustomize/overlays/default | kubectl apply -f - + +# Or via make +make kustomize-deploy +``` + +### Deploy with Istio + +```bash +kubectl kustomize config/kustomize/overlays/istio | kubectl apply -f - + +# Or via make +make kustomize-deploy KUSTOMIZE_OVERLAY=istio +``` + +### Deploy with GKE + +```bash +kubectl kustomize config/kustomize/overlays/gke | kubectl apply -f - + +# Or via make +make kustomize-deploy KUSTOMIZE_OVERLAY=gke +``` + +### Undeploy + +```bash +make kustomize-undeploy # default overlay +make kustomize-undeploy KUSTOMIZE_OVERLAY=istio +make kustomize-undeploy KUSTOMIZE_OVERLAY=gke +``` + +## Customization + +### Change the target namespace + +Edit the `namespace:` field in the overlay's `kustomization.yaml`: + +```yaml +# config/kustomize/overlays/default/kustomization.yaml +namespace: my-namespace # ← change this +``` + +Or patch it inline from the command line: + +```bash +cd config/kustomize/overlays/default +kustomize edit set namespace my-namespace +``` + +> **Istio users:** The `cluster_name` in `overlays/istio/envoyfilter.yaml` and the `host` in +> `overlays/istio/destinationrule.yaml` embed the namespace as part of the FQDN +> (`payload-processor..svc.cluster.local`). This is handled automatically — the +> `replacements` block in `overlays/istio/kustomization.yaml` injects the overlay's +> `namespace:` value into both fields at build time, so there is nothing to edit manually and +> no risk of drift between them. + +### Change the container image + +Add an `images` override in your overlay's `kustomization.yaml`: + +```yaml +images: + - name: ghcr.io/llm-d/llm-d-inference-payload-processor + newTag: v0.3.0 +``` + +### Change the Gateway name + +Patch the `targetRefs[0].name` field in `envoyfilter.yaml` (Istio) or +`gcproutingextension.yaml` (GKE) using a strategic merge patch: + +```yaml +# overlays/istio/gateway-patch.yaml +apiVersion: networking.istio.io/v1alpha3 +kind: EnvoyFilter +metadata: + name: payload-processor +spec: + targetRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: my-custom-gateway # ← your Gateway name +``` + +```yaml +# overlays/istio/kustomization.yaml (add to existing file) +patches: + - path: gateway-patch.yaml +``` + +### Use a custom IPP config + +Add a `configMapGenerator` entry in your overlay's `kustomization.yaml` to merge your own +`PayloadProcessorConfig`: + +```yaml +configMapGenerator: + - name: payload-processor + behavior: merge + files: + - custom-ipp-config.yaml=path/to/your/config.yaml +``` + +Then update the `--config-file` arg in a Deployment patch to point to +`/config/custom-ipp-config.yaml`. + +### Multi-namespace RBAC + +The base uses a namespace-scoped `Role`/`RoleBinding`. To watch ConfigMaps across +namespaces, create an overlay that replaces them with a `ClusterRole`/`ClusterRoleBinding`: + +```yaml +# overlays/multi-namespace/kustomization.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: my-namespace + +resources: + - ../../base + - clusterrole.yaml + - clusterrolebinding.yaml + +patches: + - target: + kind: Role + patch: |- + $patch: delete + apiVersion: rbac.authorization.k8s.io/v1 + kind: Role + metadata: + name: payload-processor-configmap-reader + - target: + kind: RoleBinding + patch: |- + $patch: delete + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: payload-processor-configmap-reader +``` + +## Notes + +- This chart should only be deployed once per Gateway (same constraint as the Helm chart). +- The `base/` layer intentionally omits `metadata.namespace` so that the overlay's + `namespace:` field is the single source of truth. +- For production use, pin the image tag and consider setting resource requests/limits via + a Deployment patch. diff --git a/config/kustomize/base/configmap.yaml b/config/kustomize/base/configmap.yaml new file mode 100644 index 00000000..d4c836b2 --- /dev/null +++ b/config/kustomize/base/configmap.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: payload-processor +data: + default-ipp-config.yaml: | + apiVersion: llm-d.ai/v1alpha1 + kind: PayloadProcessorConfig + plugins: + - type: body-field-to-header + parameters: + fieldName: model + headerName: X-Gateway-Model-Name + - type: base-model-to-header + profiles: + - name: default + plugins: + request: + - pluginRef: body-field-to-header + - pluginRef: base-model-to-header diff --git a/config/kustomize/base/deployment.yaml b/config/kustomize/base/deployment.yaml new file mode 100644 index 00000000..71312f55 --- /dev/null +++ b/config/kustomize/base/deployment.yaml @@ -0,0 +1,62 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: payload-processor +spec: + replicas: 1 + selector: + matchLabels: + app: payload-processor + template: + metadata: + labels: + app: payload-processor + spec: + serviceAccountName: payload-processor + containers: + - name: payload-processor + image: ghcr.io/llm-d/llm-d-inference-payload-processor:main + imagePullPolicy: IfNotPresent + args: + - --config-file + - /config/default-ipp-config.yaml + - --v=3 + - --tracing=false + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + ports: + - name: grpc + containerPort: 9004 + - name: grpc-health + containerPort: 9005 + # Conservative starting point for a request/response processing + # sidecar; tune based on observed load (payload size, RPS) before + # running in production. + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + memory: 512Mi + # Port 9005 serves the standard gRPC health protocol + # (grpc.health.v1), see cmd/runner/health.go. + readinessProbe: + grpc: + port: 9005 + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + grpc: + port: 9005 + initialDelaySeconds: 15 + periodSeconds: 20 + volumeMounts: + - name: config-volume + mountPath: /config + volumes: + - name: config-volume + configMap: + name: payload-processor diff --git a/config/kustomize/base/kustomization.yaml b/config/kustomize/base/kustomization.yaml new file mode 100644 index 00000000..8b75a993 --- /dev/null +++ b/config/kustomize/base/kustomization.yaml @@ -0,0 +1,9 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - configmap.yaml + - serviceaccount.yaml + - rbac.yaml + - deployment.yaml + - service.yaml diff --git a/config/kustomize/base/rbac.yaml b/config/kustomize/base/rbac.yaml new file mode 100644 index 00000000..1afb4a6d --- /dev/null +++ b/config/kustomize/base/rbac.yaml @@ -0,0 +1,20 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: payload-processor-configmap-reader +rules: + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: payload-processor-configmap-reader +subjects: + - kind: ServiceAccount + name: payload-processor +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: payload-processor-configmap-reader diff --git a/config/kustomize/base/service.yaml b/config/kustomize/base/service.yaml new file mode 100644 index 00000000..972c0b75 --- /dev/null +++ b/config/kustomize/base/service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: payload-processor +spec: + selector: + app: payload-processor + ports: + # Named so Istio (and other mesh sidecars) detect the protocol from the + # port name instead of treating the traffic as opaque TCP. + - name: grpc-ext-proc + protocol: TCP + port: 9004 + targetPort: 9004 + appProtocol: HTTP2 + type: ClusterIP diff --git a/config/kustomize/base/serviceaccount.yaml b/config/kustomize/base/serviceaccount.yaml new file mode 100644 index 00000000..8af6ace9 --- /dev/null +++ b/config/kustomize/base/serviceaccount.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: payload-processor diff --git a/config/kustomize/overlays/default/kustomization.yaml b/config/kustomize/overlays/default/kustomization.yaml new file mode 100644 index 00000000..75363fd2 --- /dev/null +++ b/config/kustomize/overlays/default/kustomization.yaml @@ -0,0 +1,9 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# Override the namespace for all resources in the base. +# Change this to the namespace where you want to deploy the payload processor. +namespace: default + +resources: + - ../../base diff --git a/config/kustomize/overlays/gke/gcproutingextension.yaml b/config/kustomize/overlays/gke/gcproutingextension.yaml new file mode 100644 index 00000000..764a5f6b --- /dev/null +++ b/config/kustomize/overlays/gke/gcproutingextension.yaml @@ -0,0 +1,36 @@ +apiVersion: networking.gke.io/v1 +kind: GCPRoutingExtension +metadata: + # Namespace is set by kustomization.yaml + name: payload-processor +spec: + targetRefs: + - group: gateway.networking.k8s.io + kind: Gateway + # Update to match your Gateway name. + name: inference-gateway + extensionChains: + - name: chain1 + extensions: + - name: ext1 + authority: "myext.com" + # IPP processes full request/response bodies (FullDuplexStreamed + # below) plus model-selection and cost-computation logic, so a + # 1s timeout is too aggressive under load or with larger payloads. + # 10s is a safer starting point; tune based on observed p99 + # request latency and payload size. + timeout: 10s + supportedEvents: + - RequestHeaders + - RequestBody + - RequestTrailers + - ResponseHeaders + - ResponseBody + - ResponseTrailers + requestBodySendMode: "FullDuplexStreamed" + responseBodySendMode: "FullDuplexStreamed" + backendRef: + group: "" + kind: Service + name: payload-processor + port: 9004 diff --git a/config/kustomize/overlays/gke/healthcheckpolicy.yaml b/config/kustomize/overlays/gke/healthcheckpolicy.yaml new file mode 100644 index 00000000..13d33f34 --- /dev/null +++ b/config/kustomize/overlays/gke/healthcheckpolicy.yaml @@ -0,0 +1,18 @@ +apiVersion: networking.gke.io/v1 +kind: HealthCheckPolicy +metadata: + # Namespace is set by kustomization.yaml + name: payload-processor-healthcheck +spec: + default: + logConfig: + enabled: true + config: + type: "GRPC" + grpcHealthCheck: + portSpecification: "USE_FIXED_PORT" + port: 9005 + targetRef: + group: "" + kind: Service + name: payload-processor diff --git a/config/kustomize/overlays/gke/kustomization.yaml b/config/kustomize/overlays/gke/kustomization.yaml new file mode 100644 index 00000000..a7a49f3b --- /dev/null +++ b/config/kustomize/overlays/gke/kustomization.yaml @@ -0,0 +1,11 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# Override the namespace for all resources in the base. +# Change this to the namespace where you want to deploy the payload processor. +namespace: default + +resources: + - ../../base + - gcproutingextension.yaml + - healthcheckpolicy.yaml diff --git a/config/kustomize/overlays/istio/destinationrule.yaml b/config/kustomize/overlays/istio/destinationrule.yaml new file mode 100644 index 00000000..eb6dc7f5 --- /dev/null +++ b/config/kustomize/overlays/istio/destinationrule.yaml @@ -0,0 +1,19 @@ +apiVersion: networking.istio.io/v1 +kind: DestinationRule +metadata: + # Namespace is set by kustomization.yaml + name: payload-processor +spec: + # The namespace segment below is a placeholder; it is replaced at build + # time by the `replacements` block in kustomization.yaml to match the + # overlay's `namespace:`. + host: payload-processor.default.svc.cluster.local + trafficPolicy: + tls: + mode: SIMPLE + # IPP serves TLS with a self-signed certificate generated at startup + # (see internal/tls/tls.go) rather than one issued by a shared/trusted + # CA, so Istio cannot validate it against a CA bundle. This setting + # is intentional for intra-cluster traffic to the sidecar; do not set + # this on a DestinationRule that targets an externally-facing host. + insecureSkipVerify: true diff --git a/config/kustomize/overlays/istio/envoyfilter.yaml b/config/kustomize/overlays/istio/envoyfilter.yaml new file mode 100644 index 00000000..338423d2 --- /dev/null +++ b/config/kustomize/overlays/istio/envoyfilter.yaml @@ -0,0 +1,40 @@ +apiVersion: networking.istio.io/v1alpha3 +kind: EnvoyFilter +metadata: + # Namespace is set by kustomization.yaml + name: payload-processor +spec: + targetRefs: + - group: gateway.networking.k8s.io + kind: Gateway + # Update to match your Gateway name. + name: inference-gateway + configPatches: + - applyTo: HTTP_FILTER + match: + context: GATEWAY + listener: + filterChain: + filter: + name: "envoy.filters.network.http_connection_manager" + patch: + operation: INSERT_FIRST + value: + name: envoy.filters.http.ext_proc.payload-processor + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.ext_proc.v3.ExternalProcessor + failure_mode_allow: false + allow_mode_override: true + processing_mode: + request_header_mode: "SEND" + response_header_mode: "SEND" + request_body_mode: "FULL_DUPLEX_STREAMED" + response_body_mode: "FULL_DUPLEX_STREAMED" + request_trailer_mode: "SEND" + response_trailer_mode: "SEND" + grpc_service: + envoy_grpc: + # The namespace segment below is a placeholder; it is + # replaced at build time by the `replacements` block in + # kustomization.yaml to match the overlay's `namespace:`. + cluster_name: outbound|9004||payload-processor.default.svc.cluster.local diff --git a/config/kustomize/overlays/istio/kustomization.yaml b/config/kustomize/overlays/istio/kustomization.yaml new file mode 100644 index 00000000..2fe78252 --- /dev/null +++ b/config/kustomize/overlays/istio/kustomization.yaml @@ -0,0 +1,39 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# Override the namespace for all resources in the base. +# The namespace segment embedded in the EnvoyFilter cluster_name and the +# DestinationRule host (below, via `replacements`) is derived from this +# field automatically, so it never drifts out of sync. +namespace: default + +resources: + - ../../base + - envoyfilter.yaml + - destinationrule.yaml + +# Inject the overlay namespace into the FQDN segments of the EnvoyFilter's +# cluster_name and the DestinationRule's host, so changing `namespace:` +# above is the single source of truth and can't silently drift. +replacements: + - source: + kind: Service + name: payload-processor + fieldPath: metadata.namespace + targets: + - select: + kind: EnvoyFilter + name: payload-processor + fieldPaths: + - spec.configPatches.0.patch.value.typed_config.grpc_service.envoy_grpc.cluster_name + options: + delimiter: "." + index: 1 + - select: + kind: DestinationRule + name: payload-processor + fieldPaths: + - spec.host + options: + delimiter: "." + index: 1 diff --git a/pkg/config/loader/configloader_test.go b/pkg/config/loader/configloader_test.go index c8041201..0ac52573 100644 --- a/pkg/config/loader/configloader_test.go +++ b/pkg/config/loader/configloader_test.go @@ -73,8 +73,8 @@ func TestLoadRawConfiguration(t *testing.T) { configText: successConfigText, want: &configapi.PayloadProcessorConfig{ TypeMeta: metav1.TypeMeta{ - Kind: "PayloadProcessorConfig", - APIVersion: "llm-d.ai/v1alpha1", + Kind: configKind, + APIVersion: configAPIVersion, }, Plugins: []configapi.PluginSpec{ {Name: testRequestProcType, Type: testRequestProcType}, @@ -91,8 +91,8 @@ func TestLoadRawConfiguration(t *testing.T) { configText: "", want: &configapi.PayloadProcessorConfig{ TypeMeta: metav1.TypeMeta{ - APIVersion: "llm-d.ai/v1alpha1", - Kind: "PayloadProcessorConfig", + APIVersion: configAPIVersion, + Kind: configKind, }, Plugins: []configapi.PluginSpec{ { diff --git a/pkg/config/loader/defaults.go b/pkg/config/loader/defaults.go index 15d1a30a..bec14736 100644 --- a/pkg/config/loader/defaults.go +++ b/pkg/config/loader/defaults.go @@ -31,11 +31,16 @@ import ( "github.com/llm-d/llm-d-inference-payload-processor/pkg/framework/plugins/requesthandling/profilepicker/single" ) +const ( + configAPIVersion = "llm-d.ai/v1alpha1" + configKind = "PayloadProcessorConfig" +) + func loadDefaultConfig() *configapi.PayloadProcessorConfig { return &configapi.PayloadProcessorConfig{ TypeMeta: metav1.TypeMeta{ - APIVersion: "llm-d.ai/v1alpha1", - Kind: "PayloadProcessorConfig", + APIVersion: configAPIVersion, + Kind: configKind, }, Plugins: []configapi.PluginSpec{ { diff --git a/test/integration/body_mutation_test.go b/test/integration/body_mutation_test.go index 8d088aef..0146b5e5 100644 --- a/test/integration/body_mutation_test.go +++ b/test/integration/body_mutation_test.go @@ -62,7 +62,7 @@ func TestBodyMutation(t *testing.T) { baseModelToHeaderPlugin := &basemodelextractor.BaseModelToHeaderPlugin{AdaptersStore: basemodelextractor.NewAdaptersStore()} h := NewHarnessWithPlugins(t, ctx, []requesthandling.RequestProcessor{plugin, baseModelToHeaderPlugin}, []requesthandling.ResponseProcessor{}) - body := map[string]any{"prompt": "hello"} + body := map[string]any{bodyFieldPrompt: "hello"} bodyBytes, _ := json.Marshal(body) reqs := []*extProcPb.ProcessingRequest{ @@ -88,8 +88,8 @@ func TestBodyMutation(t *testing.T) { } wantBody, _ := json.Marshal(map[string]any{ - "prompt": "hello", - "injected": "test-value", + bodyFieldPrompt: "hello", + "injected": "test-value", }) wantResponses := []*extProcPb.ProcessingResponse{ { @@ -101,7 +101,7 @@ func TestBodyMutation(t *testing.T) { SetHeaders: []*envoyCorev3.HeaderValueOption{ { Header: &envoyCorev3.HeaderValue{ - Key: "Content-Length", + Key: headerContentLength, RawValue: []byte(strconv.Itoa(len(wantBody))), }, AppendAction: envoyCorev3.HeaderValueOption_OVERWRITE_IF_EXISTS_OR_ADD, diff --git a/test/integration/hermetic_test.go b/test/integration/hermetic_test.go index 4cf90726..628da313 100644 --- a/test/integration/hermetic_test.go +++ b/test/integration/hermetic_test.go @@ -79,7 +79,7 @@ func TestBodyBasedRouting(t *testing.T) { SetHeaders: []*envoyCorev3.HeaderValueOption{ { Header: &envoyCorev3.HeaderValue{ - Key: "Content-Length", + Key: headerContentLength, RawValue: []byte("50"), }, AppendAction: envoyCorev3.HeaderValueOption_OVERWRITE_IF_EXISTS_OR_ADD, diff --git a/test/integration/util.go b/test/integration/util.go index 008065b0..916fad38 100644 --- a/test/integration/util.go +++ b/test/integration/util.go @@ -23,6 +23,11 @@ import ( extProcPb "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" ) +const ( + headerContentLength = "Content-Length" + bodyFieldPrompt = "prompt" +) + // --- Response Expectations (Streaming) --- // ExpectHeader asserts that the payload processor set the specific model header and cleared the route cache. @@ -37,7 +42,7 @@ func ExpectHeader(modelName, baseModelName string, contentLength string) *extPro SetHeaders: []*envoyCorev3.HeaderValueOption{ { Header: &envoyCorev3.HeaderValue{ - Key: "Content-Length", + Key: headerContentLength, RawValue: []byte(contentLength), }, AppendAction: envoyCorev3.HeaderValueOption_OVERWRITE_IF_EXISTS_OR_ADD, @@ -68,7 +73,7 @@ func ExpectHeader(modelName, baseModelName string, contentLength string) *extPro // The payload processor buffers the body to inspect it, then sends it downstream as a single chunk (usually). func ExpectBodyPassThrough(prompt, model string) *extProcPb.ProcessingResponse { j := map[string]any{ - "max_tokens": 100, "prompt": prompt, "temperature": 0, + "max_tokens": 100, bodyFieldPrompt: prompt, "temperature": 0, } if model != "" { j["model"] = model