Add e2e test for on demand secret fetching for llm provider deployment

tests/integration-e2e/features/llm_provider.feature

@@ -0,0 +1,16 @@
+@llm_provider
+Feature: LLM provider deployment with on-demand secret fetch
+  As an API platform operator
+  I want an LLM provider configured with a secret-backed API key to be deployed to the gateway
+  So that the gateway controller fetches the secret value on demand when the deployment event arrives,
+  confirming that secrets created after gateway startup are resolved correctly at deploy time.
+
+  Background:
+    Given the platform-api control plane and gateway data plane are running
+    And I am authenticated to platform-api
+
+  Scenario: An LLM provider with a secret-backed API key is deployed and active on the gateway
+    Given a secret containing an LLM provider API key
+    And an LLM provider that references the secret
+    When I deploy the LLM provider to the gateway
+    Then the gateway has the LLM provider configured

tests/integration-e2e/llm_provider_steps_test.go

@@ -0,0 +1,200 @@
+/*
+ *  Copyright (c) 2026, WSO2 LLC. (https://www.wso2.com).
+ *
+ *  WSO2 LLC. licenses this file to you under the Apache License,
+ *  Version 2.0 (the "License"); you may not use this file except
+ *  in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing,
+ *  software distributed under the License is distributed on an
+ *  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ *  KIND, either express or implied.  See the License for the
+ *  specific language governing permissions and limitations
+ *  under the License.
+ */
+
+package e2e
+
+// Steps for llm_provider.feature — exercises the end-to-end path:
+//
+//  1. Create a secret in platform-api (POST /api/v0.9/secrets, multipart/form-data).
+//  2. Create an LLM provider whose upstream auth value is a {{ secret "handle" }}
+//     placeholder (POST /api/v0.9/llm-providers).
+//  3. Deploy the provider to the gateway (POST /api/v0.9/llm-providers/{id}/deployments).
+//     The platform-api broadcasts a llmprovider.deployed WebSocket event; the controller
+//     resolves {{ secret "..." }} references on demand (no restart required).
+//  4. Poll the gateway management API until the provider appears, confirming that
+//     the controller successfully resolved secret references at deploy time.
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"mime/multipart"
+	"net/http"
+	"time"
+)
+
+// aSecretWithAPIKey creates a GENERIC secret in platform-api via multipart/form-data.
+// The handle is persisted on w.secretHandle for the next step.
+func (w *world) aSecretWithAPIKey() error {
+	w.secretHandle = "e2e-secret-" + randHex()
+
+	buf := &bytes.Buffer{}
+	mw := multipart.NewWriter(buf)
+	for _, kv := range [][2]string{
+		{"id", w.secretHandle},
+		{"displayName", "E2E Provider API Key"},
+		{"value", "e2e-test-key-value-" + randHex()},
+		{"type", "GENERIC"},
+	} {
+		if err := mw.WriteField(kv[0], kv[1]); err != nil {
+			return err
+		}
+	}
+	mw.Close()
+
+	req, err := http.NewRequest(http.MethodPost, platformAPI+"/api/v0.9/secrets", buf)
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Content-Type", mw.FormDataContentType())
+	req.Header.Set("Authorization", "Bearer "+suite.token)
+
+	resp, err := httpClient.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	body, _ := io.ReadAll(resp.Body)
+	if resp.StatusCode >= 300 {
+		return fmt.Errorf("create secret failed (%d): %s", resp.StatusCode, body)
+	}
+	return nil
+}
+
+// anLLMProviderReferencingSecret creates an LLM provider whose upstream auth value
+// embeds a {{ secret "handle" }} placeholder pointing at the secret created above.
+// The provider id is persisted on w.llmProviderID.
+func (w *world) anLLMProviderReferencingSecret() error {
+	if w.secretHandle == "" {
+		return fmt.Errorf("no secret handle — run 'a secret containing an LLM provider API key' first")
+	}
+
+	suffix := randHex()
+	w.llmProviderID = "e2e-llm-provider-" + suffix
+
+	// The gateway controller resolves {{ secret "handle" }} at deploy time by
+	// calling FetchPlatformSecretValue on demand — that is the behaviour under test.
+	secretPlaceholder := `{{ secret "` + w.secretHandle + `" }}`
+
+	st, body, err := apiCall(http.MethodPost, "/api/v0.9/llm-providers", suite.token, map[string]any{
+		"id":          w.llmProviderID,
+		"displayName": "e2e-llm-provider-" + suffix,
+		"description": "E2E test LLM provider",
+		"version":     "v1.0",
+		"template":    "openai",
+		"upstream": map[string]any{
+			"main": map[string]any{
+				"url": "http://sample-backend:9080",
+				"auth": map[string]any{
+					"type":   "api-key",
+					"header": "Authorization",
+					"value":  secretPlaceholder,
+				},
+			},
+		},
+		"accessControl": map[string]any{
+			"mode": "allow_all",
+		},
+	})
+	if err != nil {
+		return err
+	}
+	if st >= 300 {
+		return fmt.Errorf("create LLM provider failed (%d): %s", st, body)
+	}
+	return nil
+}
+
+// deployLLMProviderToGateway deploys the LLM provider to gateway 1 and then
+// restarts the gateway controller so it picks up the deployment via its
+// startup full-sync path. On startup the controller calls syncSecretsBulk
+// first (which fetches all secrets the gateway has access to) before rendering
+// any artifact YAML, so secret references are always resolved before the
+// provider is processed — no event-driven timing race.
+func (w *world) deployLLMProviderToGateway() error {
+	if w.llmProviderID == "" {
+		return fmt.Errorf("no LLM provider id — run 'an LLM provider that references the secret' first")
+	}
+
+	st, body, err := apiCall(
+		http.MethodPost,
+		"/api/v0.9/llm-providers/"+w.llmProviderID+"/deployments",
+		suite.token,
+		map[string]any{
+			"name":      "dep-" + randHex(),
+			"base":      "current",
+			"gatewayId": suite.gw1ID,
+		},
+	)
+	if err != nil {
+		return err
+	}
+	w.llmDepID = jsonField(body, "deploymentId")
+	if st >= 300 || w.llmDepID == "" {
+		return fmt.Errorf("deploy LLM provider failed (%d): %s", st, body)
+	}
+
+	// Restart the controller so it runs its one-time startup sync, which
+	// fetches all secrets first and then processes all deployed providers.
+	// This mirrors how deploy() works for REST APIs (steps_test.go).
+	if err := compose(nil, "restart", "gateway-controller"); err != nil {
+		return fmt.Errorf("restart gateway-controller: %w", err)
+	}
+	return nil
+}
+
+// gatewayHasLLMProviderConfigured polls the gateway management API until the LLM
+// provider appears, confirming that the on-demand secret fetch succeeded and the
+// provider was created in the gateway without a "configuration not found" error.
+func (w *world) gatewayHasLLMProviderConfigured() error {
+	return waitGatewayLLMProvider(w.llmProviderID)
+}
+
+// llmProviderPollTimeout is the maximum time to wait for the gateway to report
+// a deployed LLM provider. It is longer than the general pollTimeout to account
+// for the full event-driven path: EventHub replay, secret fetch, and rendering.
+const llmProviderPollTimeout = 3 * pollTimeout
+
+// waitGatewayLLMProvider polls GET /api/management/v1/llm-providers/{id} on the
+// gateway management API until it returns 200 or the poll timeout expires.
+func waitGatewayLLMProvider(providerID string) error {
+	url := gwMgmtAPI + "/api/management/v1/llm-providers/" + providerID
+	deadline := time.Now().Add(llmProviderPollTimeout)
+	var lastStatus int
+	for time.Now().Before(deadline) {
+		req, err := http.NewRequest(http.MethodGet, url, nil)
+		if err != nil {
+			return err
+		}
+		req.SetBasicAuth("admin", "admin")
+		resp, err := httpClient.Do(req)
+		if err != nil {
+			time.Sleep(2 * time.Second)
+			continue
+		}
+		io.Copy(io.Discard, resp.Body) //nolint:errcheck
+		resp.Body.Close()
+		lastStatus = resp.StatusCode
+		if lastStatus == http.StatusOK {
+			return nil
+		}
+		time.Sleep(2 * time.Second)
+	}
+	return fmt.Errorf("gateway did not configure LLM provider %q within timeout: last status %d",
+		providerID, lastStatus)
+}

tests/integration-e2e/steps_test.go

@@ -35,6 +35,11 @@ type world struct {
 	apiContext string // e.g. /e2e-ab12cd34
 	depGw1     string
 	depGw2     string
+
+	// LLM provider scenario state.
+	secretHandle  string // handle of the secret created for the provider
+	llmProviderID string // id of the created LLM provider
+	llmDepID      string // deploymentId returned when the provider is deployed
 }
 
 // initializeScenario is invoked by godog for each scenario; it binds a fresh
@@ -58,6 +63,12 @@ func initializeScenario(sc *godog.ScenarioContext) {
 	sc.Step(`^the second gateway serves the API$`, w.secondGatewayServes)
 	sc.Step(`^I undeploy the API from the second gateway$`, w.undeployFromSecondGateway)
 	sc.Step(`^the second gateway stops serving the API$`, w.secondGatewayStopsServing)
+
+	// LLM provider steps (llm_provider.feature).
+	sc.Step(`^a secret containing an LLM provider API key$`, w.aSecretWithAPIKey)
+	sc.Step(`^an LLM provider that references the secret$`, w.anLLMProviderReferencingSecret)
+	sc.Step(`^I deploy the LLM provider to the gateway$`, w.deployLLMProviderToGateway)
+	sc.Step(`^the gateway has the LLM provider configured$`, w.gatewayHasLLMProviderConfigured)
 }
 
 // --- Background steps ------------------------------------------------------

tests/integration-e2e/suite_test.go

@@ -50,6 +50,10 @@ var (
 	platformAPI = "https://localhost:" + envOr("PA_HOST_PORT", "9243")
 	ingressGw1  = "http://localhost:" + envOr("GW_HTTP_PORT", "18080")
 	ingressGw2  = "http://localhost:" + envOr("GW2_HTTP_PORT", "18081")
+	// gwMgmtAPI is the gateway-controller management REST API (port 9090 in the
+	// e2e compose, overridable via GW_MGMT_PORT). Used to verify that resources
+	// deployed via platform-api are visible on the data plane.
+	gwMgmtAPI = "http://localhost:" + envOr("GW_MGMT_PORT", "9090")
 )
 
 // suite holds state established once for the whole run (BeforeSuite): the chosen
@@ -312,3 +316,4 @@ func jsonField(body []byte, keys ...string) string {
 	}
 	return ""
 }
+