diff --git a/gateway/configs/config-template.toml b/gateway/configs/config-template.toml index d11d5ebb66..2291942f24 100644 --- a/gateway/configs/config-template.toml +++ b/gateway/configs/config-template.toml @@ -212,6 +212,18 @@ connect_timeout_ms = 5000 server_header_transformation = "OVERWRITE" server_header_value = "WSO2 API Platform" +# HTTP Connection Manager (downstream) timeouts +# A value of zero disables any of these timeout settings. +[router.http_listener.timeouts] +# Max duration for the entire downstream request +request_timeout = "0s" +# Max duration to receive the complete request headers +request_headers_timeout = "0s" +# Idle timeout for a single HTTP stream/request +stream_idle_timeout = "5m" +# Idle timeout for the downstream connection +idle_timeout = "1h" + [router.policy_engine] host = "policy-engine" port = 9001 diff --git a/gateway/gateway-controller/api/management-openapi.yaml b/gateway/gateway-controller/api/management-openapi.yaml index f3926b0181..d38c74cd5c 100644 --- a/gateway/gateway-controller/api/management-openapi.yaml +++ b/gateway/gateway-controller/api/management-openapi.yaml @@ -4194,6 +4194,8 @@ components: description: List of API-level policies applied to all operations unless overridden items: $ref: "#/components/schemas/Policy" + resilience: + $ref: "#/components/schemas/Resilience" operations: type: array description: List of HTTP operations/routes @@ -4258,6 +4260,25 @@ components: pattern: '^\d+(\.\d+)?(ms|s|m|h)$' example: 5s + Resilience: + type: object + description: > + Backend/route timeout configuration. Maps to Envoy RouteAction timeouts. + Can be set at the API level (applies to all routes) and/or the operation level + (applies to that operation's route). When set at both levels, the operation-level + value takes precedence. When unset, the gateway's global route timeout defaults apply. + properties: + timeout: + type: string + description: Maximum time for the entire route (request to upstream response). "0s" disables the timeout. + pattern: '^\d+(\.\d+)?(ms|s|m|h)$' + example: 15s + idleTimeout: + type: string + description: Per-route stream idle timeout (overrides the listener stream idle timeout for this route). "0s" disables the timeout. + pattern: '^\d+(\.\d+)?(ms|s|m|h)$' + example: 0s + Upstream: type: object oneOf: @@ -4313,6 +4334,8 @@ components: description: List of policies applied only to this operation (overrides or adds to API-level policies) items: $ref: "#/components/schemas/Policy" + resilience: + $ref: "#/components/schemas/Resilience" Policy: type: object @@ -5795,6 +5818,13 @@ components: enum: [deployed, undeployed] default: deployed example: deployed + resilience: + $ref: '#/components/schemas/Resilience' + description: > + API-level backend/route timeout configuration. Applies to all routes generated + for this LLM Provider (the routes that forward traffic upstream). Supported at the + API level only - LLM routes are synthesized by the gateway, so there is no + operation-level override. UpstreamAuth: type: object @@ -6085,6 +6115,13 @@ components: enum: [deployed, undeployed] default: deployed example: deployed + resilience: + $ref: '#/components/schemas/Resilience' + description: > + API-level backend/route timeout configuration. Applies to all routes generated + for this LLM Proxy (the routes that forward traffic upstream). Supported at the + API level only - LLM routes are synthesized by the gateway, so there is no + operation-level override. SecretConfigurationRequest: type: object diff --git a/gateway/gateway-controller/pkg/api/management/generated.go b/gateway/gateway-controller/pkg/api/management/generated.go index 5dbc799012..678131e055 100644 --- a/gateway/gateway-controller/pkg/api/management/generated.go +++ b/gateway/gateway-controller/pkg/api/management/generated.go @@ -504,6 +504,9 @@ type APIConfigData struct { // Policies List of API-level policies applied to all operations unless overridden Policies *[]Policy `json:"policies,omitempty" yaml:"policies,omitempty"` + // Resilience Backend/route timeout configuration. Maps to Envoy RouteAction timeouts. Can be set at the API level (applies to all routes) and/or the operation level (applies to that operation's route). When set at both levels, the operation-level value takes precedence. When unset, the gateway's global route timeout defaults apply. + Resilience *Resilience `json:"resilience,omitempty" yaml:"resilience,omitempty"` + // SubscriptionPlans List of subscription plan names available for this API SubscriptionPlans *[]string `json:"subscriptionPlans,omitempty" yaml:"subscriptionPlans,omitempty"` @@ -787,6 +790,9 @@ type LLMProviderConfigData struct { // Deprecated: this property has been marked as deprecated upstream, but no `x-deprecated-reason` was set Policies *[]LLMPolicy `json:"policies,omitempty" yaml:"policies,omitempty"` + // Resilience Backend/route timeout configuration. Maps to Envoy RouteAction timeouts. Can be set at the API level (applies to all routes) and/or the operation level (applies to that operation's route). When set at both levels, the operation-level value takes precedence. When unset, the gateway's global route timeout defaults apply. + Resilience *Resilience `json:"resilience,omitempty" yaml:"resilience,omitempty"` + // Template Template name to use for this LLM Provider Template string `json:"template" yaml:"template"` Upstream LLMProviderConfigData_Upstream `json:"upstream" yaml:"upstream"` @@ -960,6 +966,9 @@ type LLMProxyConfigData struct { Policies *[]LLMPolicy `json:"policies,omitempty" yaml:"policies,omitempty"` Provider LLMProxyProvider `json:"provider" yaml:"provider"` + // Resilience Backend/route timeout configuration. Maps to Envoy RouteAction timeouts. Can be set at the API level (applies to all routes) and/or the operation level (applies to that operation's route). When set at both levels, the operation-level value takes precedence. When unset, the gateway's global route timeout defaults apply. + Resilience *Resilience `json:"resilience,omitempty" yaml:"resilience,omitempty"` + // Version Semantic version of the LLM proxy Version string `json:"version" yaml:"version"` @@ -1214,6 +1223,9 @@ type Operation struct { // Policies List of policies applied only to this operation (overrides or adds to API-level policies) Policies *[]Policy `json:"policies,omitempty" yaml:"policies,omitempty"` + + // Resilience Backend/route timeout configuration. Maps to Envoy RouteAction timeouts. Can be set at the API level (applies to all routes) and/or the operation level (applies to that operation's route). When set at both levels, the operation-level value takes precedence. When unset, the gateway's global route timeout defaults apply. + Resilience *Resilience `json:"resilience,omitempty" yaml:"resilience,omitempty"` } // OperationMethod HTTP method @@ -1255,6 +1267,15 @@ type Policy struct { Version string `json:"version" yaml:"version"` } +// Resilience Backend/route timeout configuration. Maps to Envoy RouteAction timeouts. Can be set at the API level (applies to all routes) and/or the operation level (applies to that operation's route). When set at both levels, the operation-level value takes precedence. When unset, the gateway's global route timeout defaults apply. +type Resilience struct { + // IdleTimeout Per-route stream idle timeout (overrides the listener stream idle timeout for this route). "0s" disables the timeout. + IdleTimeout *string `json:"idleTimeout,omitempty" yaml:"idleTimeout,omitempty"` + + // Timeout Maximum time for the entire route (request to upstream response). "0s" disables the timeout. + Timeout *string `json:"timeout,omitempty" yaml:"timeout,omitempty"` +} + // ResourceStatus Server-managed lifecycle information for a resource type ResourceStatus struct { // CreatedAt Timestamp when the resource was first created (UTC) @@ -5748,281 +5769,285 @@ func HandlerWithOptions(si ServerInterface, options StdHTTPServerOptions) http.H // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+y9/XLjNrY4+Cq42qm6diLK8lcn7albt9y209Gk3a3xR3L3xt40REIWxhTBEKBtJeOq", - "fYh9wn2SX+GLBEmQomRKlhzNH5O2SAIHwPk+B+f82XLJOCQBChhtHf3Zou4IjaH453G/d0KCIb47hQzy", - "H8KIhChiGInHLgkYemL8nx6iboRDhknQOmp9gBSBELIRGJIIQN8Hx/0eiEjMEAVb45gyQBmMGHjEbAR2", - "2iAggEUQ+zi4A9SHdLTdAdcUgb89oIhiEgBGABoPkAfYCAH9Iw7En2KiLdS567TBToSgh4M7x8eU7SSf", - "R4gS/wFRPk72lYfdTne702q30BMchz5qHbXsY7TarTF8+oSCOzZqHe11u+3WGAf67912K4SMoYgv//+5", - "udn5FTp/HDv/23Xe/3Zz49zc7Nx+8yv//fZvrXaLTUI+EWURDu5az+2Wh0KfTMYoYJcMMiR3dAhjn7WO", - "1EPktdq5bT5FFEfIA+nXfFsZAg74T/3Rf4ItNdI2IBH4zzhInnTALyMUAIoY3xbzSVvsKz8zTEGExuQB", - "eWAYkbE8w4gf1nCIXTCIGXAFhsQR5FC1xVf3aELbAAYeCImPXYwogBECYYQoisRYJAIhYShgGPogQukK", - "xFEE8bh19Ku58BS41q15VsYrxU3FNPTh5DMcoyKK/hiPYeDwk4YDX641gGOksHOAwPXFJ2cYYRR4/gQ4", - "gAT+BPiIHzFtgyAeD8Q/aAhdRNtgNAlHKKBtwAGNqEsipHbAI4xyEiCPyNvO4NmFRDPwCVPGAchi2G4l", - "hqXodXPj/HZz0wG331oxi9OrOBla3AMxMRmCH6+u+iB9cUcSaqvdwgyNxXd/i9CwddT6v3ZSVrGj+MTO", - "F/0hn26Mg578aDcBBkYRnPCHGhnKITnu9xwfPSDfQJww9DEnfCIYSQomiAMfUQrIA4oi7HkoqAtxn48t", - "IMpDSONBAlbfh1WbZr4KQh8GAn8ogA8Q+wKnOJKzEabqbJOD/7X1kfgcYy+x/4AijtAJ2IXzy0MYh5RF", - "CI6LgKV7p9/JkmarnWPfY4iDaVt1rafjmwMDb0Ce6n/y3G5F6PeY8yi+ajHfbbIkMvgXcpm5plM0xAGe", - "gqwRiqnY3mSVXvqZFChEfAN9wPAYkTyLqo3Y1wWwbAeixUMB4Es0hgHDbiKuyFCz1Qwb4BKolSHuh5sb", - "79ubmw7/j5WoH0aEMssencSUkTF4wBGLoQ/EWzse4RtPFTrq+e2oMHU4NZpk4BHxYlfgv5IHmXXBEHfU", - "Xx2XjFul/Ktzc+OUcC8D5WYCTX1nhUs9c14OXz38zr1lSqUUe9qJMmWQeIZ72wjnuN/7CU2Ku3OKGMQ+", - "5RgHAy2RzU34k59Oz2sdtUxdh2+Jo9ARhlgMzf8R/ra7t39w+O6779934cD10HDWv/n6IgQZ8o65RrPX", - "3XvndA+c7u7Vbvdov3vU7f5v+soHMa03xnxbMkK8dT4B/RTrflKLCnGEKB84iH2/3Qrku+OJk2KoIzeA", - "kjhy+UOfuNDnPzDIYsrncxl+QEJKZShD7VN+h68D/HuMQBgPfOwC7HFNZohRZBA5YCPIxB/3aMIVKUgp", - "cTFfoWBTGaQsO4YCRehzyQP0EQUcVZCnj1uyQnF6XPEa4qc8dTZyrAUAjXPOw3iFx4gyOA7BI1c89T4J", - "YCEFd3oJGUBLcGVIojEU2jFkyOGMvgKYD5YN6xXOLKYoAo8jkgJigpjdPYWdL9I5hb5pcGWxEVscCo64", - "D9hDXhuMY8ZfzmqONjKoVh0LgBpUkwfzjD+Ckq8nJ7bFaQvgITfVUPLCdv6ovnO6u/youvycqo6KD8cX", - "1jpiUYysAHJeDP0LNLQR4Jl6DCI0RBEKXAR6p/ndzEDn+iT2OG2NOTNw3n//3btD2xEG1rPj5gCFQ2TS", - "euHsYMyIk2KPsJgMjGgDPFbn2ebY5gFIpfUawgiOEUNRdkNtLMw453f7mWPeL0iwrvP+9tstJ/nn9jd2", - "Kau4YkGDEb+bLE2sUvBObkzqI9o2bDbNWPWzrLmmnxZBUHy4AIL4PQeCMZ1i21zEPpB7xTpCIWszEyfv", - "VYvwQEplyfQTqEymZvIUk4qSXSyX0yf8Q0yCC/R7jKggPEMgl0otm0iyioAvWu0NfYgDh2sTyaE9QD+W", - "zEYfjJRKAQcRk6BzE/SGIGU7wm6RUsT3uTks0BUHlCHo8eNQWM7tVwgC9AhIgDo3wZUSd/qzEaQj5IEB", - "GpIIAcpIBO9QB+jXXBjwt3AAYDABklHcBFtjHOBxPAb774A7ghF0udWtXEICMr4QBXtwlyzJn6Ss+ybQ", - "jojOTZAhqifxP+eRkj0haUMfMj6z4ArqofwPl5gmfb17OR/tgN4QDAgbAfVhLxBegmQY5SjR55D+zuA9", - "olySu8jj7K5TlJK7e073+zmkZAJK5Ro8ZT9ZmGwWP/WLFr1UD2Gio57AXM9+NwETBwzdoUjYiQEu0SoA", - "f2QZT3EJilwSeFQep/JtjEgc8f96cML/84jQvXiBBGxEc04m+Uo16xDAtdPF2/hAEzJNEBknAYx8j6uV", - "ibXL8UiQqfgigi6njTCOQkIRFQ4sRaB3kKFHmBILBZhRQB4DwDdbQKDnjaB7j4O7PA3VlaWY0hhFFcoX", - "lR5cEjFurnOFWbHXhAMJikkJQvjhYIgFaYOsrL0J+GCUq1VqRM2GoOuikCFPDBYQluF0KEJ8HwOiv4oQ", - "X4Hmi3m1OWUYHnqQX9iWPob0HnnHJbz6XDy1uAYEW+Rbr/SG5AA7N0FfAQ0GE7ltChDxnVCpU54YRshR", - "zNfGBIX6/80333zzNPnju+/f19eDelZTR59TdmshUK5nU2nSR2LX9pei8TzXENE0JAFFORmdSt6N+Vxm", - "Po8RpfAOSYekwOaUSGnsuojSYez7E6GzjSEOcHAnqeSfMWGwdfTeGFZ9UKUDVXnwlH/EhMo4z+kAFmjC", - "DnGeRi70WwlB/85fTDg5N/FMrH9vE3apRmy4rtR2TJNFid6ql12ulH7ClJnYbttm8c9aLtN0wwue9VmW", - "024xwqB/QuLAJvD5MxWCUUEDweMyCkRxS8up/gJpbbZEOS+g34xa30ZVWzNVrQpXHohbkBE5b3oVs1GG", - "6lRWsyT6vw45vhlYD33/y7B19GsdQs9btM+3WTgUl759brdO+PYMsQsZqmY5bvpifb5jjJ6M3BAT+jBh", - "toilZEID/lC42X0fGJCDIfZRhiHt7e0evrcy+llYXeUUNXmeba8sqR1WeD7bIKE6EYNDZAK0a1suLvem", - "G1ri1vV173Q74V/GbBleenjYRd8fdLsO2ns/cA52vQMHfrf7zjk4ePfu8PDgoNvtdmexS4y9AfIdcPoZ", - "bHEwhjiiTAAC8BAM4sDLe2VPPv/X+QScHLe/8P9+ie5ggP+QWREn/3V9aTUSUk6R83tJrATCzyFFgzTy", - "9BeZiQ2o49AnkNsI3Bq8PL0EsSDw6fzGru5zxVEr+mWHMJ44rgjHOS60jkzY8ZBN225kiC/+d81Nl9J0", - "19l7B7rvjrrfHe29qy1MDXagpU/CDFAUkSgrWyo4BY0leVWuUL20SIyaQu/XAjkMZl/Keosr6Z+dOyhw", - "Ccet/+kcdt+b+LBFtzvgBAbAJQGDOADj2Gc49DNIQ7MuK4f/78PZx95ncHJ2cdX7oXdyfHUmfr0Jznu9", - "0/+5Ojk5vv/l7vix9+H4rveP458+da8/fju++In96/y4+/Hk8vePl73B/uk/zz6cPF4fn59dP538cfyP", - "D3eff74JOp3OTSBGO/t8aplhBte/5E6ZcI2xrA44VylDsXwRuhGhNC8ScqvPEc0ciT+d32pFpbNUK1Zo", - "0wbOOL6XywNBDrQs0ow8riZiT5KverdmlsXPyYcCBJvYLuWSP+K7kcp5EZMC83GGkMwEEBPWoYC+rv4l", - "mUIj2tfZE4ugsK1Tj0px23HmWXbx/7j88rkPpSc5QlT6kSIwQtBDkcRWRrRMlQ4jRu6R0ugz2/O3TswB", - "7eAgjNkVf8nK5Xyl+RZh+UU40RgBQxx4xlSG7DJ0/BBOOB/imr0AttVu/R6jaNKHEVR5GCP57wz/TT+r", - "3v8EzLa5f7ZD+PTp/Fjw9BMSsIj4Frx/clFYkpGkNl+/wJfPVw6l5HblkGBMPFSXFi5IzNCZHtFKCny0", - "YuqXdcokRub75PE36PsigTSYiH/msijVr1NTXPjIJTupsuoKW6iZqhEF9MeOSyhzBpAiz4kgQz4eC5us", - "gHMcF+rbAQkY/GymZGuZGVh1A4Npuo6Eq3IrBAwW45CNiJddkj6pj2dXrXar/+VS/Oea///p2aezqzP+", - "5/HVyY+tdutL/6r35TOX/T+eHZ+22q1vDCjK8wZFhFn6dDwPS2WybwAmo/BFDgMuxdYqzjrAwZ3KuVYB", - "a5r41qVXGlOZujnpABGmwIwifyjSX0BmPOLGOt+3sIWh2jkjJ9sdQSZO3Ec6ia/6xMQY7WS7kx0oOzLp", - "tY6q8t1hnldMQcUsb3luZxPmdXr3TiGvu4H0+WxCOwlRAPGMGexbpSns2/+9Pknsnz6dA322M2ezr1UK", - "e2alil+ls/xy+WUPfAlRcNxL3lpIwvmdTwbQ75emen8Uz8EWDLFU3baLud5Kgz7+9MnM94YUkAABOoIc", - "X6hLQtQGiGszwsBVOQbJB7lE8s7Ls8OToctX96VkdgmuyGKnIXK5Qi4onO4oBmWuBHJrGciNnB3+Lxko", - "rQvJJuKHEXJFIM4qBE7P+hdn3G46BQ6IqbHBehc64JJh3wcjEpCYH80WUzFcqX65IjODkeKX27UXleoX", - "luUwNA59q9F6pZ4k6jBfQJKXb1JMhlgSflnAbjP9vp6n1Migr/ficcxVl9t5UstLFzRvjnlx6p+NjGu5", - "q0nKAeemOLjrgMs4DEnEKGfkgQcjD6jUbHFDog1oPFBJ6W3Ozh+x77npW1QZ1EPC9VZw8cOJI+Q+hgET", - "04pZo9jnmPeL+lZyZ5kcIO/aaKekj4bMGXNofThAvr4o9o2Z+71tCY93JBKo1HBTcB7uV/DJrZubb25u", - "Ov9O+eXt1n8fZbjn7Z/d9rvdZ+ON7f++uelsf6t+uf1zr/083bAvSyRPqCGTSZ7VXWopQUZsqB6ql42Q", - "hAfaeY0qNbLrzXCBZAhapgWKeEOeMqIHFDljGMA75AEfD5E7cX0k02VoB/RJGPuCOclrgcLfIZgrF6Rf", - "An8i2aDFlXabT6D/WdNnS2XUdMz0kM4jJXscfXaEfXGPA6911Prkj03xixj0lK6p4u4iL03ink4DlvHg", - "ELlWJdQ0UX817ItfpSFxq9Vpiw7ND8R4n5sfxuvc2PPrvbTzp/hvz3sW2ySt1NSsNFVfrY3u8FOgrJCj", - "UNRRUvaeMuYMH46ltaCcCUctzkFJpDylKR3xw5FBUOkBOWp9QDBCEaD3zoTEkaNf4Hw+8ltHrRFjIT3a", - "2cmyA36eJneW3DXjMrJla+wdXHW/O9rbPdrd/99WO1H7qt7BXhlCyMly6qNy9ZeP+PxcQef2tNQNmm/Q", - "PIPmtlScn8sUlcQe0UqvcsAmwkrbSVMxK2My1cDDgj4jEbMUQPE4hcfE38zUWcS2BPRSTK8SZOf6PQPl", - "ZxKtwkORVwmMo1ALNiBSE00R/VeGLj2z1NcfbwS+lRNepZqZhSMqZpiwAQMzUmamnPO50EDiwE9f/I1p", - "N37qtU886M92biTTg8YhmzKLfGnaDEmuXMloT6nj10nedWyDKo6nkB1Rds65sAU8wZ0rAJKHP9/XIk1j", - "ysaId6r3ZVY1QagAedSYQ9Rr3Cur5FFEsCqytEav5vBXZezzjAWWYGS15VWMF+QQeJ5VWDB3vmGyyDrf", - "GJL/ncMwxMEdnUFOpMw4N4SNFOaBLUcRsw9RYdDWlFKL0lk3nHrDqWfTdBNOtqqabgJguaabYH2ZxmuQ", - "xWtovhkZtkDdN8cxFycu36iwst2IkE/0bWfhLk0d8WO10bkqXEpXb00V+ispzpLdmA/raBHt9Iiz5UNM", - "Qe5CPOW5FNynSe1qcJvg9hKD20+Ttx/ZDsUyl12iLfXSPU1miQJtouWbaPmqRssTb3st2fE06Rve+bki", - "0qGink04+q8Yjg4NN/oUvWLOgHPu843zOe/SkLKr1I8h6TPjxMgFsRxNwmUxLPEw5ZO/3mbZTHkcc4lx", - "1NxSXhg/LUG6Bh1R63Vqs4YFnyarHBN8mtjdJE8Tm2/kabJ8h0jGFmvWF2KI/GIKtApCTwEwm8s25dKn", - "uD8LNB0C3x+D0JbEVpIIUS2WsFe20gyMhYXqKHtpSd40a1/H021p+CoE/+cUKMVTG5znJ/2+cBJZjiK6", - "Exn0tKISmK+K5ibvCs1IXjxLEwcSnTJXHMAcs3jxK61+rHQ9Pcl8t0urvk63ynLxiI1QlBlBGsPqi2S0", - "ASE+gvJSDWY+qti1Udb8FK9PB9N2Y8R2pHl9vHKby2DKXnSb7SKjpYKhdERa78DPtleBcaJyUGsxo/l3", - "TxLEDJ6oTNWlk77yl6hXgLolYriT0AOKJmwk3ZHr4QZKl/Wm3UDpMjU6FaLGZ+bhNX+3YXoB+xRGexn7", - "l7tVJFXVdwGnEsQy2Owe5fOTvjaLrBU2QuSWqn18c0qVPvNG/6HTfefsfp+pImupzkH8meC+IvIWVlVJ", - "/cXm9Of5wtUIgQF071HgCcwRlBeBOJLF/Liyla9dX+WESZHPtq9lFaX/0p6VsRvu5qrAr5FvJcHc6ZJy", - "Zt+K9fONbyW10s/d0G6gpxqEM3ZDJ/FqFO30jK6RtdIzkizh+b/eZng2/zPDcQ3m2Uo4JH/L5HFp0u/R", - "jgHC0X63u9TEdts+vcAvU4mwjfhl/jInPpMzJ5U6q+rQSSEUH6QA8QPNzClPeGmuHIsx05Qrx9TTZrPs", - "E9Nuio05xmN0pVwhJSOc987P9J7XtFG5SmQakUkWha1EC/6janb+mCsHokhby1p6bX7jVsNV07xtt+II", - "z2KRl687X8wwwlV1fbTeOxsO/FjqbeDrH8aBK3cIM6vrU9SRkYUe7HVr0qoSQ1koFT2FyOWyPC0s0YRf", - "g/NDa1e0mFVAmJx/NahyEEBZFLssjlDD7hMOu70SdN1qJVkCNg/FiikGk8tx/yAgLG0iZy8g8qfNWZIp", - "UpOOInR4GA0wi2A0AQEJHF2niO+w5m+y4L80FhzZw0aXs842M6oWFWFE+BodoXR0d9977w/3h463//07", - "5zv47sCB8P2es/v9u/dw7/u993uo27KlQAmj4iXr/yQGEEu/RxNH1lUNIY6kU5bI6m6ioULgAYp8Vcn7", - "uN+jHfATmlAgEl8CwpIyazK3JbcbKHjAEQmEl/KolRZxFnfMuELQUjZnKyv5rcuupLgRDDwf2XjWzJ2N", - "6vr/0m6DJZV1LMzs6qoP1MP2TLV2VIUdXXInoyrI7631iiz5jyRmKust26HuT8HvnkHoQxeNiO9Jxmf4", - "IweE3NOdP7H33MonsXW+mdNLVUghEva0qF8lGgqoLQZbqrciEoUooeeJIlfFJo3bL3Vr5ctM6cMSu1mJ", - "BmVVp9ATcmP+wgkJJLlaKybrwmmqdJZIMnT1F9AHyTCJU1vOVyGb0rMT6eXNF7nKrX32Ulf5vJu/NVn+", - "ygZdrSJYpUR7BD6eXbUBJ9U26F9ftYEk1DYQdNoGij7bgNOr8DR9o5NLZyT4TXGt5otrvRqFmjJIeAE6", - "WrH4FcZsJJPtGPJuwX/8F+BHNF/g0jKfS+zq6zx4cpyoSQZaJHE7MTfYGkYIOaIVyD2a7EjVItFLt21Y", - "UOpE/jmbwKfx7QuXEGP4LxI5QlgkPbiTzFntcH3otsHD7nYH/BD7PqD5xED91m6n2+luy8YrLMVzrvvo", - "FiER+pfQtGW/qI+qa42qCeCjyGjrPUISOGA0DBfNYHBw5/NnzB1xzBlymNL+4ZRB309dyLoxDh7DO5R3", - "FTfDOm0UkvOFWpIrS1yhRnlNyfBMFSwXE52pLWNi3T5CqusUqyzVreurk21rh8acj69eDWbTWzgrYD6k", - "LM0b2SJjzERzTf5yGoxsENh6tcshpfguSNv0qKjOFvo9hj5HzMS+4bixPV8fUMqsBa9KI80RCknE8kA1", - "F8g1fLRzHaMuU94oej3biY0d93szRSr4B5vQR+oIF1sSYrsz3I7Cdne4+e7O31L1MusZV43yHW6+OLJ9", - "tdnW/tfU9FNKnq7aImwno7JL60jrfxVvWIaQ5ld2nOs6byUKpu3F20zyZbJ/FDFHZp8ZipW4/ZWEMvRj", - "46u0oWHoqIN00v3UlWCk1JXV5SKj2Ld1QNOPkA7hccuPhOLX59vn57wPIRd60J3FC5VmaGeA/4Uj2PHQ", - "ww4VGEl3CrjD2RR20U4Sl1hWcKqMEc8dnsqxkUYCUhs63NDhitDhTCHD435vZYOFonN7NkyoySwzY0p7", - "SwsXHvd7dSOFRohQBQ1LI4W5ovNVvppSF02m20N9h0s934rNodo37pFn/aUv9WXYtugSuRFiVSmns+ZK", - "UzFiBvI+oewuQpf//AREDhU/voG8LE3pI4m8fErj3sELEyolEEu/VHuqF9a3Lqyhm7VJ5n1etxZrli6S", - "LcoIt5ZQ4EaTkOUBpXG4H9F9N9pn/2FaHOUHMq3vfnVel4B4Gv5x4dskDrYBHppmKg5cP/ZET+ANei4K", - "PWesfGSe/yISmy41N7KokfqcneScDWmVY8o1UCSrUdr2Wis4GeqbTb9QRL6qKoYCL3GC5K6aqdPITJ6c", - "0NKUjYLMayozyYrMUgUW/SSRbElpQa8vl1c7/esrsCM5A01cHx3wlU/XEajzVfuUI8TiKEDe3wFFCJTT", - "kLwTJabekbYcUDo+GBAPo3zz7LdAZlPs5l2ne5jpTits4iKMNuM39+00yp2FGEvpq0g6r0IniWzObO/0", - "rxOPoLSytGNwDoJL5p2R8i4QizB6sF2x+3iWUpywmBOyU7oCDu6Ah5QGlaHEN0g4ZfJpQ08LkzsrTEuc", - "4HsMjV9bDXsZt7d7QOthZ8HVudHTXk9Ps8ufZUWlvqj4Kw7kvXPhEAJjOAEPMJr83bA5lfnN9TRk2Jwe", - "GKEI2cNYzWmeU9qP12vEreShKd0ObRnm+r3SVDz1Qgf8QCL+RxxhNpHXelNBKreY75eOcYv++Q8omohd", - "Tq7AYcr+zvVeIcsB1OkPatcHE4BFUS0yENmj/JNUcIuZapeYyvG/lzZbfy49Ljs/L+xnT6aOJImVTAed", - "aQd8Jkz2J499H2TxXPTK9MFWQMBXEdr5Ckh0E3xN40Rf1Z3BinSKfKy6IO3nzy64hGMEIM2mDIAdfaIy", - "ATfjvrCx7epofSPg1yu7cRkPktVJY6+0GTUMca/EPW/2iTcSHXqnov8tLPbRdt8P9wbvIHJ29/YPnMN3", - "333vvIcD1/HQsMt/4r/Ytkmkx0qxZIUlfZyBSVy9P0UPfRIx6O9cXl1ud0CS8i9yt9LMUECNPbHl9rdb", - "AyzS3k5E2Q4U2UD5gFVmnHonA48mirZMEgqgP2HYpYBF0L3Hwd121azmkVXNbC6jgdmpQee6vsLxyVXv", - "5zNDAic/9D4n/7w4+/nLT2enVp3VhLHvQ+t6zPWC0IcBuL7uncp7z5BxHjvGTPCaAU6yEVODqtOaMq+o", - "MGu7EQJ/j1F2F2UPZT6zwPrgQVWpBlua1P4OlAcbUjCCdCT8oXkn9kCW6nbgwN3d23+a/DGVeiXt2eCe", - "RtQ1hatFUJpUUDsV2pw6mbZWRdvLHCpM4UbqrPmbWZZ58uX8/OzipHf8yXbw6CnE0eQK5zPDBaPd3XP2", - "d6/29o8O3x8dvq8vJzhSfi4km38kvtcgIWW02uSxZXQSfgn+GRMGLxB0R5l5ZDprMoz801KOZxQRxnz0", - "iVPWiUaR5LPdbrdrvbxnfnYdYGYarueYy+wfSRy12q1TOGm1W+ckkJcJ0nWp51Pig3q7b2ugUSP4zwea", - "jwb4ly+jg3LgcyRQQIWMSlQPk7PkUe8bZd5J1l2iQ1WSTFWn/ypyqIX7dbG7JjpXK27zpkDmz1w63Ovy", - "vkZOcV0PpA5/mfEEyikuUYGnK6YN64yL0wdtI8/BOebiAnXwalEKZONq4ZYOb8kYOAlUCOvvoux0Xzm+", - "HJHORFKfgHAcPGHK8mdEt6caik3wmym85qVHZJv+2kiDy+XuqydJEaZscbQt5T5hMLpDjBuXERqiCAWu", - "sC9JgJRfLXsf32/dPrf/zPWCGLZun2/zXoQR4drCY4TzleRgzEihipy6+ELBiDwKf8aPhDIgUwYBpsry", - "VfcfVI0mfQ9GpwJ2wFc+9lfgIR9xIqKywFMkoFAfnAUPZNIGjyPsjtQTdcfGnDGm+r6aHhy4fkwZisSQ", - "HfB1DIMY+l+Bhykc+IgCPvUYMuwa83FLSl6pp/y/PnZxrkidiiLpaptya+TYViIVulKxA4k6Ob5ACMII", - "iQv9yEugPxUX/MuqXYi8yUJSDY6QyxLsub74JGhNXPbVNfcEtKnKqYqwhBHxHPXd0WG3292BId552DON", - "AFnZYQYEt1cyhatb37RqMcZxWA4zFhhlYF6GcLN3rDmjIrG02X0CPTCAPgxcwQARY6LXSp4yB5CivjXz", - "MO1fIisSJG1MUOCFBAeMSm8spil06s6bOuPtDjj2fZ1jQJN718nr4v7bCD4gdeVTTRaiwENeJ5vumKDN", - "C2tlmPN7JiUYtdImjn7F2Z2/vmJJvp46pWnWjkaPK/W6Ub+vwsuuKVRycppDkEeE70aqNm4WQcqL41r5", - "wQeDEWwJvirra0ZMCOm2PEqXjBGV9Tk1mm1P4xHOruASU9lDuyUXY6mUK363rNH00CkJBHa73QxI33fF", - "ceMx5wj6sOVfFtu8UKHG3nB+jIOe3NzdKfUA1B3K9KBvKxjHVYpIxXtopFABlW9IgvmaJov+fhIEfJ7C", - "oCfygdDL1Pheoj9Isr9pHdKblvhvtzumN63saR/SfGEH79st1Qpj+7+3xvTf9N/jf4+2/1ZPGPwMfeyJ", - "+c+iiFhKeItYUnEhP4gQExtBBoYQ+zIgpEbKOhRD5Hb03RFroJNSeDc9vRNx8IB+25zhRBXnLTSDEuTk", - "inI0nN2qX+vtyy9o8CEi9yg6DnH9qKj51eYGYJqqkNlNa8ICZcS9d1gkL5Lk7x5B3z8ZwSBQJXVI8Jub", - "ENJvWBnbZmuC57Z6icZSrBUfyko3xYfcdhWwGtDdw+E9dLwIy0uwOeEv3hZt4OV78gdn9+h99z2XtZlf", - "9+Svt+lOi8fCsDWWGEbYTXgJX8QPEREmCSMhdvWGddRryXIXtCdtVRcIXZFyGJ6zDe0zJ1p+S+ySvwau", - "5GsgQRQgL4xFyEVi09OzeEQDStx7xJzkYbKVybNlFqa0oO4LLv6ZpHKcoH15U6x+RBhxiQ/GyMNSkBTq", - "9XAbxvdBgl95DLbjTl1GJyt2fIxIHLYKODb/IAYuzjXINK7+ISHyXPlHKWKApFcgiT6rBRQ20K48f8iM", - "UKg+nmMqtt7Upl1WuwRIdtp0FK1gDAhhlEUwVHc46HY26/Kl/Czfu6B6W/Tluey21K2Tob42tup2yrEr", - "grJpvINLwTs0ndisxJ84cEDwP6MNaUH5Mxl2XeQ9kZ/J1Kxyjr7mVJmTJLXHkB/pzZlG29m9tGnhOZVe", - "7o2sJijIX1lc+rxBgh2F41bCMD+HiSmMqAlQdvisomwK02noLyedhuv2i185ZaruEViEUUZVqjuO4rs5", - "faesSmJt6lFkXaijeA5DbtIXqFva7wPsCaecKgErzDqOBQaxb90j0SlDcwVdhWWMAxPUXctZlHYWabyr", - "bUYHmbG5bWlv2/XpbCsSXt5yMxO+wGXfbKzWzJtvl2Lq+vWNbPXNPN1G5UJe3mfUopXLQCp4MLpi0B3V", - "xyIpxybnz12fV4UXpg6nRpOomhRt1ehe6AeabVpRcjydmxun5HAoDLwBeZoZNPWdFS71zHk5fPmCoHwT", - "rfGDOm0yUmeVYWAqOWcIrWnC11R7irek+M9mbdUCIsxQlpWbd2HoTxooq1q9oowSVkOrknhZ1KpSYcyI", - "lLJzalVKl0yHz4XPFqBTXRhcqsyA0Hgzl9WoJygajFWej5ebjcfJmyn8xli1zbsE/oJll3pnZrHuph9I", - "g1fbN47KjaPy1R2Vb7k4UobAMtPmSG9pN+IKBnNTlZK4PIgHMxWPTD7ZBI8yPJlvShlDvsNsFA8c9MCX", - "Xaxal7Aqs97c5fUH3QXlqIUpjVGumlzmhTD2/d+SWC9flME9MtOXc4+PmP0YD8CZeK21vOCEZXdeFpzI", - "4mdTQvftH/AbZ+rqAPMcPTnXZbLzESH3x/3eIph5nYDclOhbW/dukdWVBV6JDe3Y4nLKsvnNQz7XICY1", - "ln8ZDwQWmm5aYySljHjzj2TmAc8/Shy8eJzn0oMqjfIc60vO2p2bZq/l0rxCFDn6JVU4P+lvsjmq5o4q", - "+2I5OcmK9er0qqhmjoY2SVJjdtxFeFNM5vTC0IydITUTWDFoqBBS6RuUkXWp36OJ5GpmsKQDzqA7AiqM", - "AjPPpAtapFbTqj5DMInbWJuULC/Q8oggGwlLbhNimRpiaYuv7tFExRo20ZbyaEu+hvJCQiybEMkmRNJk", - "iOS2XMapQkPclM3WH81VoZ2BLGWj9kRWq6ImW6LRPCPAQxEWlyAQwONxzNIWmdSPZd0I4/67NNcUsFOJ", - "7WUFWUu2pOwip5ESnTGm+Bh61XcoQJFwyqgiOcPY962N5lTtaGvr3HSU0IdYplFnai5tPY4ocn/Tl1e+", - "Be8OwAg9cWEc0e3OTXChrxWiJ+gycbPQReD//3//P1lqAmDGD0OkJiJ/AiDlv4gbKwFhsmuTuGooDgve", - "QRzkGylJCHbh3mDfPfAO0bvhd/D7wXu36+2iveE+PBgcuu+879D3w/ewO9h197x9dDA8hO8G37nfe+9R", - "d8i/ra51UKPkULv1aJ5iTTNUvtwLhqSANGr2NAM+OaqpKCTGsyQuSmtWFU4BP54fn6jTlC2x0jMuKajV", - "ma1w0Tunu3u12z3qzla4aGaSn0K5da8yXXxyKBxKbgC2EhbRBoKBQH1zSrdNkzVckCaRHQ/5iKEcG1HO", - "ocdyYIpXVKHL8AMS7tMHcp/XYZKns1VhmussnqchWnV9CqNaWS1LyUITL6wD1m4xwqB/Uq8Km9xaXUMt", - "FSQ5JWfPekWpKCh1+TXRRTG5B4hdfa9SLFtUE+C/psCPGAulOxUrOhb9eeV1IeWR/OXyy55QQ/XNbHCF", - "4Lhoil2cXV6J9/hihAtf9QDOKuJUl3wrjqv65Emuq/pPtyzN885FfECYFXK/Ugen6k4i2rsEXHU9au13", - "up39ltGqdMfleCPcmXKr7mwy6SKpRef7qtAAuPp0CcyPgRtHEQq4rPEJ9NJ2fMZLUvh0boKrEaIo+zm3", - "PQTFD2X/P9VF+serq/5l5oaoClGqes5J65qepzwJJ+aK0sYsYnV73W7SM0eiplGDYedfVOrgNOkoXkU2", - "xjwZehQoZHdwZDb7ud06bBAccU+sCohewBVT6OsWAeLmlqSYeDyG0UQDahyym91LBu9EmNpYuoGAnGE+", - "OYKqYMxGTkR8Ef5tQW8sCm2oZjcoEtHrkFBmu8wv7jVCEKDHPI6Brf7ZOZAcdFvfideEIkSp+TKmGhG9", - "SQDH2IW+PxEOBRKLcqAMRkxfftejFDBKwmMsuNXWvYM+EG9S4/iMQIwBXuuo5fD/fTj72PsMTs4urno/", - "9E6Or87ErzfBea93+j9XJyfH97/cHT/2Phzf9f5x/NOn7vXHb8cXP7F/nR93P55c/v7xsjfYP/3n2YeT", - "x+vj87Prp5M/jv/x4e7zzzdBp9O5CcRoZ59PLTOkcZbxxJHn7bjSnz8r/stNSoKjWZVKxCALdLi7CDqs", - "Qn8TZ+NQYUZGSX9utw6WS5DibmYGaZV6sIq8IUOZboYgGuQLz+2sTNqJEJ9WOpBtDONclJDwJ4BF+O4O", - "yfapAlKuWnBWZkoZ4R8TVUSxj+iEyrKrOVZSYAIXKMcEXixYaliW5nRqSaqz7uXpZdJpc6qZOYfe9mHC", - "bG56qbcN+EO9twqonJhIVba93cP372vpbVX0aiw/T7ArRyUJOiokbFKCWqhD9L4TJ8WtIFvDIP47gFkm", - "o4kgKztHMLgTYlP7+V8iN+XEWbmZtr4WWQG5zT3Vzk0TVOE/EkvLXLY/7KLvD7pdB+29HzgHu96BA7/b", - "feccHLx7d3h4cNCVRRZwINo8icZgOufAa+Vlkynv8obYbaNkLgtazbyMqrv5VnahtmzBzGJGIk6AKsrc", - "g+WRsAlQQBgYkjjwVpKR2Ci3GQbi+2MnjMgD9lDkMDQO/UrjT9gEnz6dA/0NSL4BEbrDlKEotfYUQ2gn", - "AXt/wmWtfGcwkZFHq9326dN5X81wlQA1hWn8IEYW3bDVJ0C5sYpp2l9CFBz3NFv4PUbRJOULWY/6shiC", - "WyiTum8tPz+jDDePtJYHyLL1dQLn5YauHV1W2+QtgTklOf6C3iag92kW4iuzeY89rVZbYShYuscGtqsY", - "vawmqItxC8Yva+KjJ/6jCLCNc45oc7IiScpSyjbMmNUArneYlplSgzJTK3NnAsd+QwMv1VK1kpmFiKxI", - "oFz+K2KyZisspjWGVNWhbQnZ+yUKdhIMfewy4KSkKZJHKByrcCP0IwS9iSycuZrMSBJdFTNokh+VKwO1", - "7YqghGUVTIwSA8HOXyplvqp9F8YDH7tmCTxlPphs02I7CGc4XgPrIAG0nv5vPwer0r0MzX8GcJZtA9hB", - "Ww9rIFg8V2jbzYCPiJWT+2ACMKOgd1qk84/Iptl/mIjWFPMRus5CKtuKlST22RWDhpWeWaiUQezTDWHW", - "IExOFuU04TVsPsTWiJlodguDtCK4HaCshW4LdXlw4RJZuqIWSqR/Idukuxq2idW/uOK2yYavTYn21eMq", - "i7RHZvBJzuuKbOvM6jZQybNtIHXhNiAREEnSU92VM7gpM3s4xVWZbOYLfZbtmuAYdxnzieW26dPXXz61", - "2vsdxfyNix1Z4ZADIS0JMxcIuSsZcSZ2ad5msM+efJNOPvVSxNxnwxGxkBovN0dlnlvPSH32eg7tPZtD", - "O0Pgs3qoM/VsFtCytJ5Xe42c2aU+7IZTt8rc2AXvdcoAlfdatEIhgGNIBF1160sZm1R2jG0bbeqTbMCk", - "WUMbcLC53umqG05QnLdgLBHxs/ecaji7F+/kzjXwb1ibLBm9WjXBAfi/j88/ccH3j8svn3Uy0iu5yHN0", - "PgV27R6XFxclw934yqf6yhNekPeVB15y52yd/eYvZn0WrXRe5/gcPvGalnfR5M7tgXFth5I9R+oNTpjT", - "L1fYGV4C9hyu8dXwiK+eI3wd/d8NUPcM3u7aTu4ZnNtvgXLnlOeL0HRq0N0KuLbXzKMtHNlm589mbYl5", - "fNozu7LXjRz/AqbHtXIa53b4VVzeszGR1XV3b/ja3B7thVkKO6rz5hRvti6Cw9+0ZehN5Xk5p/Rxv/cT", - "n7Qe45NdZ21ML9N3WAO3/oqJ3J66Fzf1wWzoq1pv4Mjj5/bMhswNaBGqDnGVQ/KjqhGg/AIKoLmIq+Ag", - "lPjTCHXpUgbioQJwrVUNuTf5GjMvUzDKxlxqAm8eiHKC0bi2jlm7K8HeXschuuXFchJJiLIDlXgk4w5c", - "g9hefQ9oBadrlvNO0Xh2/oQh/gmJEHWlx/RClFzhsCrQO+BL4CKgSrG0AWbAhQEICPBJcMeNUlUtghEz", - "9IOS3r+2S7x8rOZZ+HJYdSFQzPfUKAcnzluoanyVGZiS9G7dAt4KT3pSK6aj8XNzazNchTEbhluD4ZIo", - "wZzVVi0L7GEpOmW1Y0pDImPVumCKquAVUIZUBYKYEUdpeFyGkADVcFe9SdZkSf1cPGtalHYrj6xJ3TY/", - "4lLTP2fXbFfKCabOeX1Y7Ea9ndd5t5K67U5akLC8Us1F8k7GCfkSt0Q65NvXa5MNfhsCJDm6hl0k9nFX", - "XJhEhG3cJG9Pa0/43Wsw7Sdcs6gJf/GVrg88YTT75YGnCcim/r/exYGnyevcGniarOSVgZW4MMDP5K3d", - "FtC0PMNdgafJq18UeMJrUvNGsaEcH36aLPyGwNPEfj2As7j6dwPShO88607vDGTvB8xwHeBpstC7ADk0", - "bTIbp3ToMv3iabI6VwAK5FsF9Sb5f97k/6fJG8z8FyTbGDPLqZSzZ/8/TWZM/X+avDRdUYyQv2Hv6Afr", - "UfkmAXemJH8hOV43w78MhFeyGp8m65bb3yz91srwf5rUSu9/mjSR27/q1DmPdG5cXZlGYK+ax7/yNGUk", - "8UvUjvM42bC+P1sWv9Q0a6fwr4lAfNM2Qi5dPzGLlpmrPxOL2GTprx3XqmIYi1bpX56mX4OpGZ7fSQMJ", - "+k+T6dn5a6VdrFdW/lpoATVS8l9OXE0l49cgoaxv7uWxbklDU3Pw10Vj2OTeb3LvX8TENplJjSfeN8pf", - "K3WXlU24b4ZTL5YjvyzF/mmyya/fMNWUqb6Z5PqmtcPXSat/SwzInki/SAa0yaLfZNGvGiPdKKrNptC/", - "kpbafOp8DSdCPm/+bamnZZny6yghNmnymzT5N618T8mRb5wrj92wXnb8+Um/33hyPIlU3rQ9NpLOWT8r", - "/vykn82KL9bTP5dv9U1e3HxOfArIcnPi03nLc+LRA4ombMTHept58YvOTD+0ZaaP3bA/Y3K6wvBXTE43", - "aGylc9MzvEBzwISMF5eark8on5leEonSry8oS9yKL80oQlOGXmp0p4QsiiiUnM6mH2rdNO+UZt5QqrdB", - "do3xhpx6NEOmd4KVdRO9DfBf1FotXXPS7bRzk1U8UtHv8MWZesgK54Dboa6XCp6cxqtlgldDsGy7KIFm", - "PfLAF0Lb1VngyQ5VJ4Hr117UvTRPuetCr/OI78bVkynE9jpJ4WtCXxzXM4juNaxY18wBT2ColwK+EFEp", - "HfVLJb2/mG3QfUXbYNOP9C3wqwrW0bTWHyHKHBjiKS7RC0TZcb+3RIeonrG+O/S43yt3hF4gKG7Di9Uc", - "93uLc4ZyMJbrBuUzljtAI7lyx8eixMXb7CbarEmm6aGWX1Mhqs2TWdOZujCHZ0JDK+3uNChdszb+k0Dr", - "hfk61aQ1XZ36jBejzajRm9FfCoMt1ZuZEEMRJ/SOb9yXdd2XfLfekOMyJaKmyDyjwNR2Wia0X9dlmQL+", - "IjNMsRu7r9KU0iJXZU28lWVw1/NX6pN4NXdlJQDLtk40MGvirGyenqtclQnVVjsq1Vsv8lMOSaQJdn3I", - "tJ5UbkCzqCaj1/FDrgflcDw2sdhrVuOt6YTUENTzQTYr++zOxwUT1RtU2LvLVNg3PsU3wHvKGcFC9fG5", - "a0vUZlP8+9kKSkxjUklVCXUjXkD0JvSANSkysT7SvKrExMtJ64W1JcpICFypSg+YAgj295zBhCEQwcBL", - "7huiwCWedPGP0BP0kIvH0G+DMEJD/IQ86Zb4CkMc/va1A64pSgjoJzSR9WUngAQmWSlWjQAOXDLmDEhf", - "oJajsRGm4j52iQ9upnsq02jcVvVi3bWSTQGMTQGMt8Rgq+pLNMpcK9SWFSwr0SgflOC9ChecrejENLA2", - "1Sc2HG3lOVqBSTSqIC67vERjjGjlWI70eLwKy9nUm9jUm1gu6+QbtDa3hkv5GdcR0/v/nmRsy1cRG6vp", - "UGm8hxF6wCSm2orXygEMOGqFPnS1iS43pgEbv6KQxNsxzGcvNPGmZMSm4sSm4sRbU7jLikw07kCgyI0Q", - "K49zXOioAkw8xtD3AWUk4lgmv+6AC8TiKKDqB4NPSi8pidlNwLkRdFks1i5eExxdep4pcuMIswkI4ygk", - "FFEZbS0GTS4VwAukOjlF3XiD2oMk/mKjvd3l4dd1wM+dRPgP5AEn30YtYV0rnVpLkzPWmK5OvT6il8ce", - "LjnqUqViKEREgRtNQtGRjAGuMEmFRT3tnYJxTJlwfQl1oHMT8MfKCqXG5zHlKhETyg7my9LP+OYnHWEH", - "aEgiBEIUUUwZClxkw3bpSJQrX1AKrxx8AdeRKgduyAuv9BdZ/0N6zgWACT5dJnQoPevyroJUsWW6/M/q", - "BsNR604pqlz7CX3IhiQadx4p2eu4ZLzzsNtqt+5xwI8lOZAxYtCDTOyFvocBGRxAipwQUvpIIkFnNERu", - "EQ37hLK7CF3+8xMYQxwA/SlIPm1nrnUctU71G31z8CS1UG3BMWsdtfa6e++c7q7TPbza7R7td4+63f/l", - "Cp1nhbHdUlZm+bfP4tRecPbydCVKS2vIxiXkp6sRB/kAU4PXAWNMBWmTCGCl3Qwx8j26wgz+tRLAFdtM", - "w6O905XM+gaOyZ2lSloVzKGa8l8glQyda2rmdx9FY8gX6uu6BFxsqd1NssA1PXORhamMjo9g5KlPxDHc", - "BAE3/1zygKIJGCN3BANMx1LKJVKHf4s9NA4JPxHgyBFEM1YQkMARZ4cCdhMoGCKl9R10D2wCTKbcGgKs", - "qK9Zyd+W1Qy2AgIUrmyvNM0dzCi6AsIcaYpkhZfaC4KosFbE5pviK8lMb6nTyFpbqYWTCgk+12/K7KnP", - "z6fuzmX1/KtC64mE5ZQeR6gsQbwJMm9XW1NUdb4VzCcl6ozWmWiX6jVTu7wJbGqlO+KKhFIuB0jmqnAK", - "RV4H9KThpl+mYhcAIzeBGl8wEzl3G0Bw2O2qnROeOjmM9s4J8xS7QOGgjfg/IlZJ+TNQiL4qUabcKcsL", - "+m9Lu0sW06JxuB/RfTfaZ/+xfkqfRnqvgnekxrNBGOtjSi/Vh7Uu7BZVq1aGZ6kZjlvHj1/wT6V+cFVH", - "kv/zKctqOIXSUEQneqcGWYYR8TreoMMpvJPhCVg61jP8SvyWHcDCUJ4bytSrCKvTTPjGVNalmiugk6Io", - "+TPj5bgJUjeHG0cRVxYr3B1tgAI48FVTfzKGjEsOfCcx9yZghM+DIpmG6sVRWpiddsAX3zNcbIKZcksC", - "DnwEHjBUvhZTAtqkkVz5X9OXMqu4VXKhVNwm3Sw2npT6QnX36ODwFTwpK5E+MNWTIhFpI97XSbxP85zo", - "lIfmvCbxIIGLM5agxuUc8xsgvgHwAWJfSI86V3QujQH6Ys5Fxp1yk9WOQBVWubrhHQusi6+fknjxCrMD", - "NoIMeGiIA0SBiLj6eIyZNNChYJqAiTjmUGUbmWPQslsf+aNclM6Rm0aXfXmV+w55YCqZXOEgdATnFYXT", - "q/nMV/seQ4FoGr56WWTsO3/y//Rq1kUpEnXdCikWKs0ZkRZbTIL2wqz8A4vzu7AM5QdfugbyeT0KeSwS", - "LytKeoiYiywYIbJhLPhXXevj9bCuuyK8/rXqbXxe+Zu5JdgkvEbLr7lRhKVe9Y2lYvjitarClYHnlaUs", - "7bvZUJbdFl2iKjPFPM28Wrco7XG/1wbGZk4tR3uZAWimmrS9U7BllEjtnfK5ZCPF7ZKSqDDEgoIrU9Xt", - "HyZLmm+AimKsxydXvZ/PWu1W73Pyz4uzn7/8dHa6iJKsdWl7HuN+Tez6ZZj0aisHQmAZGyDuJdeuwlI0", - "1pdgqK+MkV5btPyVbXPgZKXGOpUvpVnEXpik2/nT/HMuu30ek72WWpmFbMFm+2tZ7BkggvUz31fBcq9v", - "tC8f77qvy/9fy15fI7S2GO8rYrfPbrIvBb8Xq2O9msleG51fy1JfI5qymu0N6zGPaDCIyD2KanST+QUN", - "Poh3m2kpM8V0T2fjgNU23ZPPqhvLXDLi3oOrSLaXyXy0uB4zWdiW223mjTW8nqnXi4lK9Rq+7C+14UuG", - "sFb7amoW1JQXZVF7Yf1fzOnzTWCyD1VOJAUD7OEIuZIjAcoiBEUZywFijwgF/KtL4t4jBlwf850TqQ8/", - "weE9BJI1qkqXIYoclwSBHAtgSnxxHmVulQzWLUbkm1M0k2xpH3GpLpostRbxNXPMm7Y0db04WQp9Qw1q", - "THxomiMVVaT6/WoyeFrXv2MifxNde7O7UNrHhnJ1yGFSHXIyK16DbjbV0NfraZM5rVdrbDMdimXbSxmI", - "1sS1tkiOUNnsJrNZ1Q61xghdt73JrG7d6HsGfaAp7aYG+b2O12+NKI5jfQHnvSUJ4fmbVGShqpMHby6y", - "iZ4VBbFs7V7xZkh4TTpYZE9lnftYFDxaLybIl3a1qKS5Ne1tkeMKjTMFW1XN5fOETcuLTcuL16pXWcGR", - "X8ePsuXFchJJkCQSm8Yfpbe2t9esL8dChcU05W0FW3Uskq0vlX3P1qsjA5oGatOkY8OD10grLjCJ5ajD", - "y+7h8ZfjUEnNjSVyqE1Pj01Pj9fktGXdPTaK70ubj6yY1ttc95HpTpbV6kHyF9S0k5N+E7Js03tk03uk", - "Wem2Ns1IliE/aDyol5d7GQ+aScoV+/7E6mTnqjlnSs29jAfVebm/IMhGKDLeXWg6roZnubm4xsRqv3cU", - "q09n33mUO+GgB47TJYCozxeTFGzk877VtGCJw/Vygg+XnROsCWzVE4JTRmCwQI3gi0wFlhPn84DLg3Xq", - "tBeWiyvHbywRNz/csrNwNXFYRbna+03+7Qz5t5om3lbybUJVzVF/Tv2ZKedWIeYMCbfJAl5qfOpVl6bZ", - "apmerm0NsmutQNdOqlXH8ZoZtVUgvIIRpMBZn1zaBRD4tCxatUdTU2jle03lz6pFrQnV1pXejWgh00jr", - "1VJl14KaVJ6sgdVe09pyzavxKRT17sUvSDzKeMEyCe2NKvzdZSv8KxW8W0kn51pwpCrWsHBV/kWZ+xqe", - "mmn7ckkN5ewbHGx6wv4aqg3rk6evT2Ldk/RTJ/fLSK6B9PwSwlrf3PyE9Jul/KlZ+eulxmyS8TfJ+C9j", - "u5uEpCYz8RcgESp1sNVMwF8A714Gj35Bxr2GaJNxv2G0KcGvUZpMaeb9YnTcV8i5f+NMyZJkv3CmtEmy", - "3yTZrxxz3Si0jWXYv6o222hifZV7ZOWy6t+8+mxNo19babVJo9+k0b9t46A8h35ZEkI10S8NOV0gFkcB", - "BboZu4TQ9wF0GX5A4Mfz4xPdib8D+j7EIvVaMm8KYIRAgPgG4MD1Yw95U0JSqg30ejHoBeesjAi5l/tS", - "uz2Uaqa9FRAQ6jPZ3kSkKvO2TVRunhDrpHHrWBTltnk0CRm5i2A4wq64n0KRG0ea4kYw4sJAdoXfGpJo", - "DNkR+Po4osj97Sv4Frw74PoScEcwots3gc7nusMPqrqzyuqSmlqWbqWyxgkfeeCbb0jgom++0SqexnSu", - "xt0EkrgpI+pyTTJSGtBSQEIq/voq//wKEvpO/A/8+U3wVRHnaAxdh+/lVx0Ks8a/HiV9AHEhBFB8F0AW", - "R4jKLJrKGNil7kG/bqxmIVk7KZdpNuo1Zehlp+/bYKnoiyFxV8vprPolaSdB6CFGPjdHfF9wOxiGCHK5", - "B2AwAcOYY6UKeUfgDrGEkDqv7YD4a4bMjjVjUoFzZN49yNw3UMmwmAJ1K2oFMyoySQyGJFusIKtSKXf+", - "lP+YGkHro2gM+ab4ExChMXlA1JAbHXCmnQ+a1XvIxw8owuI9yFQ0jZ9PcqK+D/B4jDwMGfInwipJZQNI", - "7ZWp9x/WUUZU+gvUFiWXMTIA3WE2igeO2mg7MOmpLqBtngRuhe4kaItSbdtaXU8QsBu84HVIv6a7UbgD", - "TcJXamHqH/QnhhwmAZLil/iJGkwZCelNoKk7uDPUQZMdyA85o0z1XlPj5fquRbuNAxQIrRx5NuXS4ml8", - "w8yjxNe4VAbSfXW90OaUS7Er0Qw1Jmc1Q3gHcdDZ8LWZHGVbcsu3l8TkOBjc7MZsIihXfHccs1Hr6Ndb", - "jpISahtZfyIu9IEaTczcbsWR3zpqjRgLj3Z2fP7CiFB29L77vrsDQ7wzTsDcedhtFWnxlLj3KNr5KR6g", - "KEAMUaNWQX74OxmKcfgxRsT3UVQ6z22yZ/kJTy6uT0HC56QmrBvs0ZSkbT33itDbBjs/6fcj8oSRMdr5", - "SR/wHyfVw8mHOonh6tMlcFHEWacrXCh89B+vrvqXIA5lfyvAlcahwuN0upP0q9nh//TpnMP6gD0UgSs0", - "Dn0+TMY/bqzM/vbLJq0117xTPE2mjT/tlGyDp05eNZb6ITPS7fP/CQAA//8bQcMxDVsCAA==", + "H4sIAAAAAAAC/+y9+3LjNtIo/ir49NuqtRNRlm+Tsbe+2tLYzkQ79ozWl+T7beSTgUjIwpoiGQK0rcy6", + "6jzEecLzJKdwI0ESpCiZkiVH+8dmLJJAA+h7N7q/NWx/HPge8ihpHH9rEHuExpD/s9PrnvjeEN+dQgrZ", + "D0HoByikGPHHtu9R9ETZPx1E7BAHFPte47jxARIEAkhHYOiHALou6PS6IPQjigjYGkeEAkJhSMEjpiOw", + "0wSeD2gIsYu9O0BcSEbbLXBDEPjLAwoJ9j1AfYDGA+QAOkJA/Yg9/iefaAu17lpNsBMi6GDvznIxoTvx", + "5yEivvuACBsn/crDbqu93Wo0G+gJjgMXNY4b5jEazcYYPp0j746OGsd77XazMcae+nu32QggpShky/9f", + "/f7Or9D6o2P9q20d/dbvW/3+zu13v7Lfb//SaDboJGATERpi767x3Gw4KHD9yRh59IpCisSODmHk0sax", + "fIicRjOzzaeI4BA5IPmabStFwAJ/VR/9FWzJkbaBH4K/Rl78pAV+GSEPEETZtuhPmnxf2ZlhAkI09h+Q", + "A4ahPxZnGLLDGg6xDQYRBTbHkCiEDKom/+oeTUgTQM8Bge9iGyMCYIhAECKCQj6WH4LAp8ijGLogRMkK", + "+FF40bhx/Ku+8AS4xq1+Vtor+U3FJHDh5DMcozyK/hSNoWexk4YDV6zVg2MksXOAwM3luTUMMfIcdwIs", + "4HvuBLiIHTFpAi8aD/g/SABtRJpgNAlGyCNNwAANie2HSO6A41PCSMB/RM52Cs8uBZqBc0woAyCNYbul", + "GJagV79v/dbvt8Dt90bMYvTKT4bk94BP7A/BT9fXPZC8uCMItdFsYIrG/Lu/hGjYOG78fzsJq9iRfGLn", + "i/qQTTfGXld8tBsDA8MQTthDhQzFkHR6XctFD8jVECcIXMwI3+eMJAETRJ6LCAH+AwpD7DjIqwpxj43N", + "IcpCGCKCXYw8G00b4zJ587nZINEgXk7PhWWbrb8KAhd6HO8IgA8QuxwXGXHQESYSJ2KE+bXx0XcZpl9h", + "9wGFjBDi5ebOPbuyKCA0RHCcByzZc/VOmqQbzQzbH0PsTdueGzUd2xzoOQP/qfon/CB+jxhvY6vm893G", + "S/IH/0Y21dd0iobYw1OQPEQR4dsbr9JJPhOCyOffQBdQPEZ+lrVVJoibHFimA1FiJQfwFRpDj2I7FnP+", + "ULHjFPtgkquRYgoP/b7zfb/fYv8xMoOHkU+oYY9OIkL9MXjAIY2gC/hbO47PNp5IdFTzm1Fh6nByNMH4", + "Q9+JbI7/Uo6k1gUD3JJ/tWx/3Cjke61+3yrgehrKzQSa/M4Il3xmvRy+avideUuXZgn2NGMlTCPxFNc3", + "EU6n1/2EJvndOUUUYpcwjIOekuT6Jnxjp9N1GscNXUdiW2JJdIQB5kOzfwS/7e7tHxy+++H9URsObAcN", + "Z/2brS9EkCKnwzShvfbeO6t9YLV3r3fbx/vt43b7X8krH/i0zhizbUkJ/8bFBPQSrPskFxXgEBE2sBe5", + "brPhiXfHEyvBUEtsAPGjkMmEhuvb0GU/UEgjwuazKX7gMiBNGXKfsjt84+HfIwSCaOBiG2CHaUBDjEKN", + "yAEdQcr/uEcTpoBBQnwbsxVyNpVCyqJjyFGEOpcsQB+Rx1AFOeq4BSvkp8cUtiF+ylJnLceaA1A75yyM", + "13iMCIXjADwyhVXtEwcWEnCnlpACtABXhn44hlyrhhRZjNGXAPPBsGHd3JlFBIXgceQngOggpndPYueL", + "dFWup2pcmW/EFoOCIe4DdpDTBOOIspfTGqeJDMpVzhygGtVkwTxjj6Dg6/GJbTHaAnjITDwUv7CdPaof", + "rPYuO6o2O6eyo2LDsYU1jmkYISOAjBdD9xINTQR4Jh+DEA1RyPQ30D3N7mYKOtv1I4fR1pgxA+vo/Q/v", + "Dk1H6BnPjpkRBA6RTuu5s4MR9a0Ee7ilpWFEE+CxPM8mwzYHQCKs3gCGcIwoCtMbamJh2jm/208d835O", + "grWto9vvt6z4n9vfmaWs5Io5DYb/rrM0vkrOO5kRqo5oW7P1FGNVz9JmnnqaB0Hy4RwI/PcMCNp0km0z", + "Efvg30vWEXBZm5o4fq9chHtCKgumH0OlMzWdp+hUFO9isZw+YR9i37tEv0eIcMLTBHKh1DKJJKMI+KLU", + "3sCF2LOYNhEf2gN0I8Fs1MEIqeQxELHvtfpedwgStsPtFiFFXJeZ0RxdsUcogg47DonlzO6FwEOPwPdQ", + "q+9dS3GnPhtBMkIOGKChHyJAqB/CO9QC6jUbeuwt7AHoTYBgFH1va4w9PI7GYP8dsEcwhDaz1qUriUPG", + "FiJh9+7iJbmThHX3PeXAaPW9FFE98f9Zj8Tf45I2cCFlM3OuIB+K/zCJqdPXu5fz0RboDsHApyMgP+x6", + "3LsQDyMdLOockt8pvEeESXIbOYzdtfJScnfPar+fQ0rGoJSuwZH2k4HJpvFTvWjQS9UQOjqqCfT17Ldj", + "MLFH0R0KuZ3o4QKtArBHhvEklyDI9j2HiOOUPpGRH4Xsvw6csP88InTPX/A9OiIZ55R4pZx1cOCayeJN", + "fKAOmcaJjJEARq7D1MrY2mV4xMmUfxFCm9FGEIWBTxDhji9JoHeQokeYEAsBmBLgP3qAbTaHQM0bQvse", + "e3dZGqoqSzEhEQpLlC8iPL9+SJm5zhRmyV5jDsQpJiEI7r+DAeakDdKytu+xwQhTq+SIig1B20YBRQ4f", + "zPNpitOhELF99Hz1VYjYChRfzKrNCcNw0IP4wrT0MST3yOkU8OoL/tTgGuBskW291BviA2z1vZ4EGgwm", + "YtskIPw7rlInPDEIkSWZr4kJcvX/u+++++5p8scP74+q60Fdo6mjzim9tRBIl7WuNKkjMWv7S9F4niuI", + "aBL4HkEZGZ1I3o35XGQ+jxEh8A4JhyTH5oRISWTbiJBh5LoTrrONIfawdyeo5J+RT2Hj+EgbVn5QpgOV", + "efCkf0SHSjvP6QDmaMIMcZZGLtVbMUH/zl6MOTkz8XSsPzIJu0Qj1lxXcjumyaJYb1XLLlZKzzGhOrab", + "tpn/s5LLNNnwrJt0puU0G9Sn0D3xI88k8NkzGbqRwQbO41IKRH5Li6n+EilttkA5z6HfjFrfRlVbM1Wt", + "DFcefDsnIzLe9DJmIw3VqaxmSfR/EzB807Aeuu6XYeP41yqEnrVon2/TcEguffvcbJyw7RliG1JUznLs", + "5MXqfEcbPR65Jib0YUJNkU7BhAbsIXezuy7QIAdD7KIUQ9rb2z08MjL6WVhd6RQVeZ5prwwpIUZ4Ppsg", + "ISqBg0GkA7RrWi4u9qZrWuLWzU33dDvmX9psKV56eNhG7w/abQvtHQ2sg13nwII/7L6zDg7evTs8PDho", + "t9vtWewSbW+AeAecfgZbDIwhDgnlgAA8BIPIc7Je2ZPP/30xASed5hf23y/hHfTwHyKb4uS/b66MRkLC", + "KTJ+L4GVgPs5hGgQRp76IjWxBnUUuD5kNgKzBq9Or0DECXw6vzGr+0xxVIp+0SGMJ5bNw3GWDY0j+7Qz", + "pNO2G2nii/1dcdOFNN219t6B9rvj9g/He+8qC1ONHSjpEzMDFIZ+mJYtJZyCRIK8SlcoX1okRk2h9xuO", + "HBqzL2S9+ZX0zi4s5Nk+w63/aR22j3R82CLbLXACPWD7HoXYA+PIpThwU0hD0i4ri/3vw9nH7mdwcnZ5", + "3f2xe9K5PuO/9r2Lbvf0f65PTjr3v9x1HrsfOnfdf3Q+nbdvPn4/vvxE/33RaX88ufr941V3sH/6z7MP", + "J483nYuzm6eTPzr/+HD3+ee+12q1+h4f7ezzqWGGGVz/gjulwjXaslrgQqYaReJFaIc+IVmRkFl9hmjm", + "SBhq/VYpKp2mWr5CkzZwxvC9WB5wciBFkWbkMDURO4J85bsVsyx+jj/kIJjEdiGX/AnfjWTOC58U6I9T", + "hKQngOiwDjn0VfUvwRRq0b7OnmgIuW2deFTy245Tz9KL/8fVl889KDzJISLCjxSCEYIOCgW2Ul/JVOEw", + "ov49khp9anv+0ooYoC3sBRG9Zi8ZuZwrNd88LL9wJxr1wRB7jjaVJrs0HT+AE8aHmGbPgW00G79HKJz0", + "YAhlHsZI/DvFf5PPyvc/BrOp75/pEM7PLzqcp5/4Hg1914D3TzYKCjKS5OarF9jy2cqhkNy2GBKMfQdV", + "pYVLP6LoTI1oJAU2Wj71yzhlHCNzXf/xN+i6PPHUm/B/ZrIv5a9TU1zYyAU7KbPxcluomKoWBXTHlu0T", + "ag0gQY4VQopcPOY2WQ7nGC5UtwNiMNjZTMnW0jOwqgYGk3QdAVfpVnAYDMYhHflOeknqpD6eXTeajd6X", + "K/6fG/b/p2fnZ9dn7M/O9clPjWbjS++6++Uzk/0/nXVOG83GdxoUxXmDPMIsfDqOg4Uy2dMAE1H4PIcB", + "V3xrJWcdYO9O5mrLgDWJfevCK42JSPmctAAPU2BKkDvk6S8gNZ5vRypPOLeFgdw5LZfbHkHKT9xFKomv", + "/MT4GM14u+MdKDoy4bUOy/LkYZZXTEHFNG95bqYT7VVa+E4uH7yGtPt0IrwfIA/iGTPftwpT37f/vj7J", + "7+fnF0Cd7cxZ8GuV+p5aqeRXySy/XH3ZA18C5HW68VsLSVS/c/0BdHuFKeIf+XOwBQMsVLftfI641KA7", + "5+d6njgkwPcQICPI8IXYfoCaADFthhu4Mscg/iCTgN56eVZ5PHTx6r4UzC7A5dnvJEA2U8g5hZMdyaD0", + "lUBmLQOxkbPD/yUFpXEh6QT+IEQ2D8QZhcDpWe/yjNlNp8ACEdE2WO1CC1xR7Lpg5Ht+xI5mi8oYrlC/", + "bJ6ZQf38l9uVF5XoFzVm+1M0DlyjsXstn8RqNFt4nM+vU1qKyGI+m6MKPW2/modVy7yv9mInYirP7Twp", + "6YULmjc3PT/1z1qmttjVOFWBcWHs3bXAVRQEfkgJEwCeA0MHyJRufrOiCUg0kMnsTSYGHrHr2MlbRBri", + "Q5/pu+DyxxOL6wsYepRPy2cNI5dh7C/yW8HVRVKBuNujnJkuGlJrzKB14QC56mLad3rO+LYhrN4SSCBT", + "ynWBe7hfwl+3+v3v+v3WfxI+e7v19+MU17391m6+233W3tj+e7/f2v5e/nL7ba/5PN0hUJSAHlNDKgM9", + "rfNUUp60mFI1VC8aIQ4rNLOaWGKcV5vhEonQtUgn5HGKLGWEDyi0xtCDd8gBLh4ie2K7SKTZkBbo+UHk", + "cqYmriFyPwlnykwAf/HciWCfBhfcbTbx/mdFnw2ZidPS00paj8TfY+izw+2Se+w5jePGuTvWxTai0JE6", + "qozX83w2gXsqfVjEkQNkG5VX3bT9VbNLfhUGyK1Sww26NzsQ7X1mtmivMyPRrfbSzjf+367zzLdJWLeJ", + "OaqrzEqL3WGnQGgutyGv2yTsPWHMKT4cCStDOiGOG4yD+qH0sCZ0xA5HBE+F5+S48QHBEIWA3FsTPwot", + "9QLj86HbOG6MKA3I8c5Omh2w89S5s+CuKVeTKctj7+C6/cPx3u7x7v6/Gs1YXSx7BztFCCEmy6idMkRQ", + "POLzcwmdm9NZN2i+QfMUmptSeH4uUlRiO0Ypy9JxGwsrZV9NxayUqVUBD3P6jEDMQgD54wQeHX9TU6cR", + "2xAITDC9TJBdqPc0lJ9JtHLPRlYl0I5CLliDSE40RfRfa7r0zFJffbwR+EZOeJ1oZgaOKJlhzAY0zEiY", + "mXTqZ0IKseM/efE3qtz/ibc/9rw/m7mRSCsaB3TKLOKlaTPEOXYFoz0lDmMrftcyDSo5nkR2ROgF48IG", + "8Dh3LgFIHP58X/P0jikbw98p35dZ1QSuAmRRYw5Rr3CvqHJIHsHKyNIY9ZrDz5Wyz1MWWIyR5ZZXPs6Q", + "QeB5VmHA3PmGSSPrfGMI/ncBgwB7d2QGOZEw48wQJlKYB7YMRcw+RIlBW1FKLUpn3XDqDaeeTdONOdmq", + "aroxgMWaboz1RRqvRhavofmmZNgCdd8Mx1ycuHyjwsp0k0I8Ubekubs0ccSP5UZnqn5JXb0xVeivpDiL", + "d2M+rCN5tFMjzpZHMQW5c3GY50JwnyaVq89tguJLDIo/Td5+RDzgy1x2SbjES/c0mSUKtImyb6Lsqxpl", + "j73tlWTH06SneefnjdDPFckOJNVtwth/xjB2oLnfp+gjcwaqM59vnNZZV4iQeYX+D0GfKedHJvhlKRIu", + "in3xhwl//fU2zZ6K459LjL9mlvLCuGsB0tXowFqvU5s1nPg0WeVY4tPE7F55mph8Kk+T5TtSUjZcvT4U", + "TVXIp1zL4PUUANM5cFMumfL7ukDRIXDdMQhMyW8FCRTlYgk7RStNwZhbqIrOF5YATm4JqDi8Ke1fhu6/", + "TYGSPzXBeXHS63HnkuEowjuesU9KKo+5skhv/C7XjMRFtyThINZFM8UI9DHzF82SastS11OTzHebtezr", + "ZKsMF53oCIWpEYQRLb+IRxv4vouguMSDqYtKdm2UNlv569PBNN1QMR1pVo8v3eYimNIX62a7OGmomCgc", + "mMY797PtlaedqBjUWDxp/t0TBDGDBytV5emkJ/0s8hUgb6Vobij0gMIJHQk35nq4j5JlvWn3UbJMhU65", + "aPOZfnj136WYXmg/gdFcbv/l7hhBVdVdx4kEMWfqz+iJvjjpKbPIWNEjQHah2sc2p1Dp0ysIHFrtd9bu", + "+1TVWkM1EN+dCe5rX9z6Kivhv9i7AFm+cD1CYADte+Q5HHM45YUgCkXxQKZsZWvllzlhEuQz7WtRBes/", + "tWdlbAe7marza+RbiTF3uqSc2bdi/HzjW0ms9As7MBvoiQZhje3Air0aeTs9pWukrfSUJIt5/q+3KZ7N", + "/kxxXI15NmIOyd7SeVySLHy8o4FwvN9uLzUh3rRPL/DLlCJsLX6ZP82Jz+TMSaTOqjp0Egj5BwlA7EBT", + "c4oTXporx2DM1OXK0fW02Sz72LSbYmOO8RhdS1dIwQgX3YsztecVbVSmEulGZJx9YSoJg/8om509ZsoB", + "LwrXMJZ6m9+4VXBVNG+bjSjEs1jkxevOFk8McVkdIaX3zoYDPxV6G9j6h5Fnix3C1Oj65HVrRGEJc52c", + "pIrFUBRmRU8BspksTwpZ1OHXYPzQ2L0toiUQxudfDqoYBBAaRjaNQlSz+4TBbq48XbU6SpqA9UMxYorG", + "5DLc3/N8mjS7Mxcs+WZylqSK4iSjcB0ehgNMQxhOgOd7lqqLxHZY8TfRYEAYC5bomaPKZ6ebJ5WLiiD0", + "2RotrnS0d4+co8P9oeXsv39n/QDfHVgQHu1Zu+/fHcG993tHe6jdMKVOcaPiJes/5wPwpd+jiSXquAYQ", + "h8Ip64tqcryBg+cAglxZObzT65IW+IQmBPCEGc+ncVk3kROT2Q3kPeDQ97iX8riRFI3md9OYQtCQNmcj", + "LfmNyy6luBH0HBeZeNbMnZSq+v+SrogFlXwMzOz6ugfkw+ZMtX1kRR9V4ielKojvjfWRDHmTfkRltly6", + "I943zu+eQeBCG4181xGMT/NHDnz/nux8w85zI5v81vpuTi9VLvWI29O8XhZvYCC3GGzJHpCIF76EjsOL", + "auWbSW6/VofIbDksdcj8FErRp6g6FnpCdsReOPE9QebGys6qwJss8cWTGm31BXRBPEzsDBfzlci05Mx5", + "Onv9xbgya5+9JFc2X+cvdZbpMkFXqVhXIbEfg49n103ASLwJejfXTSAIvAk4fTeBpOsmYHTOPVTfqWTW", + "GRnFpghY/UXAXo1CddnFvQctpZD8CiM6Esl9FDm34L/+G7Ajmi/gaZjP9s1q7zx40onVKw0t4ngfnxts", + "DUOELN6y5B5NdoRKEuuz2yYsKHQ+/5xO/FP49oVJljH8tx9aXMjEPcbjTF3lqH1oN8HD7nYL/Bi5LiDZ", + "hEL11m6r3WpviwYxNMFzpjOpViYh+jfX0EVfq4+yu46sQeCiUGtbPkICOKA1ROdNa7B357Jn1B4xzBky", + "mJL+6IRC101cz6qBDx7DO5R1MdfDOk0UcpmSmdn0fK7iiZ7T5sa7LXABAy7Rz7wHfwK4ktKx41YJfkSJ", + "qCI8QDz4KNtndnpdGd3aEvoDUU2kRfb/NjuMHYlpiTKR/4T3F4pf+CsR329r4U5IRXMu/i1ppkeUSojA", + "2lw7LjFI5BFEm/oh/ZWoHOP01sjgq9CJJuIQs8VfXXQt3jYkDaHQEgPKLsjs7XhwTZHioQpMKPJQaHw3", + "vk2jdqPfaJN+g5mBzEYUI8iX0zWM2ySrIjrfb8mk2u2/b43Jf8h/xv8Zbf/FbKkWrOwCPvHmb+yFmIEw", + "JhgiuYVbkk/ywlyqC7TyyM+ygN3D+VfwbCYQPchgyFouiDFodXKFRqDbNplkg5n6q8Zuo0dIVMFxmTa+", + "dXN9sm1stZpxnlcrpq674WcFzIWEJglZW/4YU94lV1CUelAjsNWaEEBC8J2X9NuS4dIt9HsEXYZ8seOA", + "Mc/t+Rr6EmqsQFeYwhGiwA9pFqj6MiS04Mdcx6j6DdSKXgXERju97kwhQPbBJqaYRJj4lgTYHGUyo7A5", + "zqS/u/OXxP5Kh5wu5VvnbETRh17rcJ6UZ4rdHaqMEndKaKWWmAAUBlLJG4YhhF8jPc5NlbdiC8z04m0q", + "qzneP4KoJdI6NcuDX8eMY4TqsfZV0pk0sORBWsl+qtJMQi0V5R5DrWq/cUDdQZcM4TBtxg/4r8+3z89Z", + "51wmpse726die7L0E2kN8L9xCFsOetghHCPJTg53GJvCNtqJA37LivoWMeK5474ZNlJLpHdDhxs6XBE6", + "nCkWz0yzVY3CM9gy8XdFZqkZE9pbWhy+0+tWDcFrsXcZjS8MwWe6R5Q5Mwt9mKm2LdU9ktWcj6ZIRU8r", + "7JAORLzU2Wfaoitkh4iW5XLPegmB8BFTkPd8Qu9CdPXPc8CTE9nxDUT1AkIe/dDJ5grvHbwwU1kAsfRb", + "7qdqYT3jwmq66h5facnq1nzNwhuzRajPrCXk2eEkoFlASRTsh2TfDvfpf+kWR/GBtKfUDilPmOQQT8M/", + "JnzrxMEmwEPdTMWe7UYOb+69Qc9FoeeMpcj0819ExuCV4kYGNVKdsxWfsyatMky5AoqkNUrTXisFJ0V9", + "s+kXkshXVcWQ4MVOkMwdTnkaqcnjE1qaspGTeXWl/BmRWajAvDEsEr1lDej15ep6p3dzDXYEZyCx66MF", + "vrLpWhx1vqqgS4hoFHrI+RsgCIFiGhKXDfnUO8KWA8pTPPAdjLJd8N8CmU2xm3et9mGqzTS3ifMwmozf", + "zLfTKHcWYiykrzzpvAqdxLI5tb3Tv449gsLKUo7BOQgunndGyrtENMTowXR39eNZQnHcYo7JTuoK2LsD", + "DpIaVIoS3yDhFMmnDT0tTO6sMC0xgu9SNH5tNexl3N7sAa2GnTlX50ZPez09zSx/lhWV+iLjr9gTBR24", + "QwiM4QQ8wHDyN83mlOY309OQZnM6YIRCZA5j1ad5sk2auYN9tqO+lIe6dDs0Xd1Q7xXmuMoXWuBHP2R/", + "RCGmE5EJkghSscVsv1SMm6ms/IYU3+X4bikm9G9M7+WyHECVHyR3fTABmFe58wc8LZt9kghuPlPlmm8Z", + "/me6Y23ohFzYC/258LjM/Dy3n12RWxVnLFMVdCYt8NkXGUE8OyqN57zprQu2PB985aGdr8AP+97XJE70", + "dduUZJNKp8jGqnPSfv7sgis4RgCSdMoA2FEnKjLbU+4LE9suj9bXAn61ejZX0SBenTD2CrvKwwB3C9zz", + "Wq7Flpbo0D3ljaxhviG+fTTcG7yDyNrd2z+wDt/98N46ggPbctCwzX5iv5i2iSeBCbFkhCV5nIKJ17Q4", + "RQ89P6TQ3bm6vtpugfguDc9NSlKnAdH2xHRpptkYYJ4XesLr4aDQBMoHLFNH5TspeBRRNEWSkAfdCcU2", + "ATSE9j327rbLZtWPrGxmfRk1zE40OleFSzon192fzzQJHP/Q/Rz/8/Ls5y+fzk6NOqsOY8+FxvXo6wWB", + "Cz1wc9M9FQUFIGU8dowp5zUDHKfratmKjSnz8pLPpqtW8PcIpXdRNENnM3Os9x5k2XiRycZI7W9AerAh", + "ASNIRtwfmnViD0TtfAsO7N29/afJH1OpV9CeCe5pRF1RuBoEpU4Fle8K6FPH01YqMX2VQYUp3EieNXsz", + "zTJPvlxcnF2edDvnpoNHTwEOJ9c4e3WCM9rdPWt/93pv//jw6PjwqLqcYEj5OXcb46PvOjUSUkqrjR8b", + "RveDL94/I5/CSwTtUWoeke8dDyP+NNS5GoU+pS46Z5R1olAk/my33W4bb8Xqn914mOqG6wVmMvsnPwob", + "zcYpnDSajQvfE7dtknXJ51Pig2q7byugUS34zwaajwbYly+jg2LgMySQQ4WUSlQNk9PkUe0bad4J1l2g", + "Q5WSTAmFlJJDJdyvit0V0blccZs3BTJ75sLhXpX31XKK63ogVfjLjCdQTHGxCjxdMa1ZZ1ycPmgaeQ7O", + "MRcXqIJXi1Iga1cLt1R4S8TAfU+GsP7G68D3pOPL4ulMfuIT4I6DJ0xo9ozI9lRDsQ5+M4XXvPSITNPf", + "aGlwmdx9dQtEVTdLVx3cku4TCsM7RJlxGaIhCpFnc/vS95D0q6ULXbiN2+fmt0xzlmHj9vk260UY+Uxb", + "eAxxtkQjjKifK88ob4YRMPIfuT/jJ59QIFIGASbS8pX3H2TxM3VRTKUCtsBXNvZX4CAXMSIionJayKGQ", + "H/B7Vk3wOML2SD6R12H0GSOiLnTGl2lsNyIUhXzIFvg6hl4E3a/JjRo29RhSbGvzMUtK1Kog7L8utnH2", + "AlhfdwbLrRFjG4mU60r5lkDy5PglMBCEiFfKQE4M/SmvnFFURobnTeaSanCIbBpjz83lOac1cWFLFrPk", + "0CYqp6xuFIS+Y8nvjg/b7fYODPDOw55uBIiSKTMguLlEMFzdwsFli9GOw3CYEccoDfNShJsuXqBurDFM", + "c33ogAF0oWdzBogo5c2PspQ5gAT1jJmHSUMhUeoj7iuEPCfwsUeJ8MZikkAnL4XKM95ugY7rqhwDEhc0", + "iF/nF0RH8AHJO9FysgB5DnLSV9EStHlhERp9fkenBK0I4cRSr1i78xcuLcjX0y72VSlVqW44aoUxS7zs", + "ikIFJycZBHlE+G4ki06nEaS46rSRH3zQGMEW56uicG1IuZBuiqO0/TEiovCtQrPtaTzC2uVcYip7aDbE", + "YgwlqPnvhjXqHjopgcBuu50C6X2bHzceM46gDlv8ZbDNc6WfXKPtPMZeV2zursFSNVwyTg76toRxFN59", + "vTbdLuYbot0GFTSZ9/f7nsfmyQ16Ih5ol5GBE+sPguz7jUPSb/D/tttj0m+kT7vmy6Q/Qxc7fP6zMPQN", + "tfF5LCm/kB95iInfdR5C7IqAkBwp7VAMkN1Sd0eMgU5C4N309E7EwAPqbX2GE1n1OtedjZOTzes8MXYr", + "f622L7+gwYfQv0dhJ8DVo6L6V5sbgEmqQmo3jQkLhPr2vUVDcZEke/cIuu7JCHqerFXle7/ZMSH9hqWx", + "rff8eG7Kl0gkxFr+oSghlX/IbFcOqwbdPRzeQ8sJsbgEmxH+/G32sXxP/GDtHh+1j5isTf26J369TXaa", + "P+aGrbbEIMR2zEvYIn4MfW6SUD/AttqwlnwtXu6C9qQpC26ha78YBrZz2iWx1IkW3xK7Yq+Ba/EaiBEF", + "iAtjIbIR3/TkLB7RgPj2PaJW/DDeyvjZMiu+GlD3BRf/dFLpxGhf3KWuF/rUt30XjJGDhSDJFcKShSxi", + "/MpisBl3qjI6UdLmY+hHQSOHY/MPouHiXINM4+ofYiLPVIYQIgYIegWC6NNaQG4Dzcrzh9QIubL+GaZi", + "ahav22WVa+Skp01GUQrGwPcpoSEM5B0Osp3OunwpP8s2BSnfFnV5Lr0tVQvJyK+1rbqdcuySoEwa7+CK", + "8w5FJyYr8RMDDnD+p/UFzil/OsOuirwn4jORmlXM0decKjOSpPIY4iO1OdNoO72XJi08o9KLvRFlOjn5", + "S4tLnTeIsSN33FIYZufQMYX6cgKUHj6tKOvCdBr6i0mn4br54ldGmap6BAZhlFKVqo4j+W5G3ykqP1qZ", + "eiRZ5wqUXsCAmfQ56hb2+wA73Cknaytzs45hgUbsW/eIt6BRXEFVYRljTwd113AWhS17am8zndJBZuw2", + "Xdhsen1aTfOEl7fcJYgtcNk3G8s18/r7EOm6fnUjW34zTxtfsZCXN/A1aOUikAoetHYzZEc2iInLjYn5", + "M9fnZeGFqcPJ0QSqxtWQFbrnGu2mu8EUHE+r37cKDodAzxn4TzODJr8zwiWfWS+HL1sxl22iMX5Qpf9M", + "4qzSDEwp5zShNU346mpP/pYU+1kvWpxDhBnqHTPzLgjcyUvrFU/TqtJKWAWtSuBlXqtKhDH1hZSdU6uS", + "umQyfCZ8tgCd6lLjUkUGhMKbuaxGNUHeYCzzfLzcbOzEbybwa2NVNu9i+HOWXeKdmcW6m34gNV5t3zgq", + "N47KV3dUvuXiSCkCS02bIb2l3YjLGcx1VUpi8iAazFQ8Mv5kEzxK8WS2KUUM+Q7TUTSw0ANbdr5qXcyq", + "9HpzVzcfVHuh4wYmJEKZanKpF4LIdX+LY71sURr3SE1fzD0+YvpTNABn/LXG8oITht15WXAijZ91Cd23", + "f8BvnKnLA8xy9Phcl8nOR75/3+l1F8HMqwTkpkTfmqopkqiuzPGKb2jLFJeTls1vDnKZBjGpsPyraMCx", + "UHfTaiNJZcSZfyQ9D3j+USLvxeM8Fx5UYZSnoy45K3dukr2WSfMKUGipl2Rnibje/eao6juq9IvF5CQq", + "1svTK6OaOTpFxUmN6XEX4U3RmdMLQzNmhlRPYEWjoVxIpadRRtqlfo8mgqvpwZIWOIP2CMgwCkw9Ey5o", + "nlpNyhp4wThuY+zis7xAyyOCdMQtuU2IZWqIpcm/ukcTGWvYRFuKoy3ZGsoLCbFsQiSbEEmdIZLbYhkn", + "Cw0xUzZdfzRThXYGsuTNShNZLYuabEVEKNkOCjG/BIEAHo8jmvSeJW4k6kZo99+FuSaBnUpsLyvIWrAl", + "RRc5tZTolDHFxlCrvkMeCrlTRhbJGUaua+zEKGtHG3tSJ6MELsQijTpVc2nrcUSQ/Zu6vPI9eHcARuiJ", + "CeOQbLf63qW6VoieoE35zUIbgf/7v/+PKDUBMGWHwVMTkTsBkLBf+I0Vz6eirRm/asgPC95B7GU7jQkI", + "duHeYN8+cA7Ru+EP8P3gyG47u2hvuA8PBof2O+cH9H54BNuDXXvP2UcHw0P4bvCD/d45Qu0h+7a81kGF", + "kkPNxqN+ihXNUPFy1xv6OaSRsycZ8PFRTUUhPp4hcVFYs7JwCvjponMiT1P0jEvOuKCgVmu2wkXvrPbu", + "9W77uD1b4aKZSX4K5Va9ynR5bhE4FNwAbMUsogk4A4Hq5pTqKyhquCBFIjsOchFFGTYinUOPxcDkr6hC", + "m+IHxN2nD/59VoeJn85WhWmus3iehmjl9Sm0amWVLCUDTbywDlizQX0K3ZNqVdjE1qoaaokgySg5e8Yr", + "SnlBqcqv8Taj8T1AbKt7lXzZvJoA+zUBfkRpINypWNIxb3wtrgtJj+QvV1/2uBqqbmaDawTHeVPs8uzq", + "mr/HFsNd+LK5dloRJ6rkW35c2UhScF3Z2L1h6C55weMD3KwQ+5U4OGV3Et7exWOq63Fjv9Vu7Te0Xr47", + "NsMb7s4UW3VnkkmXcS0615WFBsD1+RXQPwZ2FIbIY7LG9aGT9KvUXhLCp9X3rkeIoPTnzPbgFD8UDTJl", + "e/afrq97V6kbojJEKes5x61ruo70JJzoK0oas/DV7bXbcc8cgZpaDYadfxOhg5O4VX8Z2WjzpOiRo5DZ", + "wZHa7Odm47BGcPg9sTIguh5TTKGrWgTwm1uCYqLxGIYTBah2yHZ6Lym842FqbekaAjKG+WRxqoIRHVmh", + "7/LwbwM6Y15oQza7QSGPXgc+oabL/PxeIwQeesziGNjqnV0AwUG31Z14RShclOovY6IQ0Zl4cIxt6LoT", + "7lDwI14OlMKQqsvvapQcRgl4tAU3mqp30AffmVQ4Pi0Qo4HXOG5Y7H8fzj52P4OTs8vr7o/dk871Gf+1", + "7110u6f/c31y0rn/5a7z2P3Quev+o/PpvH3z8fvx5Sf674tO++PJ1e8fr7qD/dN/nn04ebzpXJzdPJ38", + "0fnHh7vPP/e9VqvV9/hoZ59PDTMkcZbxxBLnbdnCnz8r/otNioOjaZWKxyBzdLi7CDosQ38dZ6NAYkZK", + "SX9uNg6WS5D8bmYKaaV6sIq8IUWZdoogauQLz820TNoJEZtWOJCJsWOsF3HKpiG+u0OifSqHlKkWjJXp", + "Uob7x3gVUewiMiGi7GqGleSYwCXKMIEXC5YKlqU+nVySbD19dXoVd9qcambOobd9mFCTm17obQP2UO2t", + "BCojJhKVbW/38Oiokt5WRq/a8rMEu3JUEqOjRMI6JaiBOnjvO35SzAoyNQxivwOYZjKKCNKycwS9Oy42", + "lZ//JXJTTJyWm0lveJ4VkNncU+Xc1EHl/iO+tNRl+8M2en/Qblto72hgHew6Bxb8YfeddXDw7t3h4cFB", + "WxRZwB5v88Qbg6mcA6eRlU26vMsaYre1krkoaDXzMsru5hvZhdyyBTOLGYk4Biovcw+WR8I6QJ5PwdCP", + "PGclGYmJcuthIK47toLQf8AOCi2KxoFbavxxm+D8/AKob0D8DQjRHSYUhYm1JxlCMw7YuxMma8U7g4mI", + "PBrttvPzi56c4ToGagrT+JGPzLthy0+AdGPl07S/BMjrdBVb+D1C4SThC2mP+rIYgp0rk7pvLD8/owzX", + "j7SSB8iw9VUC58WGrhldVtvkLYA5ITn2gtomoPZpFuIrsnk7jlKrjTDkLN2Ohu0yRi+qCapi3Jzxi5r4", + "6In9yANs44wjWp8sT5KilLIJM2Y1gKsdpmGmxKBM1crcmcCxW9PAS7VUjWRmICIjEkiX/4qYrOkKi0mN", + "IVl1aFtAdrREwe57QxfbFFgJafLkEQLHMtwI3RBBZyIKZ64mMxJEV8YM6uRHxcpAZbvCK2BZOROjwEAw", + "85dSmS9r3wXRwMW2XgJPmg862zTYDtwZjtfAOogBrab/m8/BqHQvQ/OfAZxl2wBm0NbDGvAWzxWaZjPg", + "I6LF5D6YAEwJ6J7m6fwjMmn2Hya8NcV8hK6ykIq2YiWJfXbFoGalZxYqpRC7ZEOYFQiTkUUxTTg1mw+R", + "MWLGm91CL6kIbgYobaGbQl0OXLhEFq6ohRLpn8g2aa+GbWL0L664bbLha1OifdW4yiLtkRl8kvO6Ipsq", + "s7oJZPJsEwhduAn8EPAk6anuyhnclKk9nOKqjDfzhT7LZkVwtLuM2cRy0/TJ6y+fWu79jmT+2sWOtHDI", + "gJCUhJkLhMyVjCgVu9RvM5hnj79JJp96KWLus2GImEuNF5sjM8+NZyQ/ez2H9p7JoZ0i8Fk91Kl6Ngto", + "WVrNq71GzuxCH3bNqVtFbuyc9zphgNJ7zVuh+IBhSAhteetLGptEdIxtam3q42zAuFlDEzCwmd5pyxtO", + "kJ83Zyyh76bvOVVwdi/eyZ1p4F+zNlkwerlqgj3w/3cuzpng+8fVl88qGemVXOQZOp8Cu3KPi4uLguFu", + "fOVTfeUxL8j6yj0nvnO2zn7zF7M+g1Y6r3N8Dp94Rcs7b3Jn9kC7tkP8PUvoDVaQ0S9X2BleAPYcrvHV", + "8IivniN8Hf3fNVD3DN7uyk7uGZzbb4Fy55Tni9B0KtDdCri218yjzR3ZeufPem2JeXzaM7uy140c/wSm", + "x410Gmd2+FVc3rMxkdV1d2/42twe7YVZCjuy8+YUb7YqgsPeNGXoTeV5Gad0p9f9xCatxvhE11kT00v1", + "HVbArb9iIran6sVNdTAb+irXGxjyuJk9MyFzDVqErENc5pD8KGsESL+ABGgu4so5CAX+1EJdqpQBfygB", + "XGtVQ+xNtsbMyxSMojGXmsCbBaKYYBSurWPW7kqwt9dxiG45kZhEEKLoQMUfibgD0yC2V98DWsLp6uW8", + "UzSenW8wwJ8QD1GXekwveckVBqsEvQW+eDYCshRLE2AKbOgBzweu790xo1RWi6C+HvpBce9f0yVeNlb9", + "LHw5rDoXKGZ7qpWD4+fNVTW2yhRMcXq3agFvhCc5qRXT0di52ZUZrsSYDcOtwHD9MMac1VYtc+xhKTpl", + "uWNKQSJi1apgiqzg5RGKZAWCiPqW1PCYDPE9VMFd9SZZkyH1c/GsaVHarTiyOnXb7IhLTf+cXbNdKSeY", + "POf1YbEb9XZe591K6rY7SUHC4ko1l/E7KSfkS9wSyZBvX6+NN/htCJD46Gp2kZjHXXFhEvp04yZ5e1p7", + "zO9eg2k/4YpFTdiLr3R94Amj2S8PPE1AOvX/9S4OPE1e59bA02QlrwysxIUBdiZv7baAouUZ7go8TV79", + "osATXpOaN5INZfjw02ThNwSeJubrAYzFVb8bkCR8Z1l3cmcgfT9ghusAT5OF3gXIoGmd2TiFQxfpF0+T", + "1bkCkCPfMqg3yf/zJv8/Td5g5j8n2dqYWUalnD37/2kyY+r/0+Sl6Yp8hOwNe0s9WI/KNzG4MyX5c8nx", + "uhn+RSC8ktX4NFm33P566bdShv/TpFJ6/9Okjtz+VafOeaRz7erKNAJ71Tz+lacpLYlfoHaUxcma9f3Z", + "sviFplk5hX9NBOKbthEy6fqxWbTMXP2ZWMQmS3/tuFYZw1i0Sv/yNP0KTE3z/E5qSNB/mkzPzl8r7WK9", + "svLXQguokJL/cuKqKxm/AgmlfXMvj3ULGpqag78uGsMm936Te/8iJrbJTKo98b5W/lqqu6xswn09nHqx", + "HPllKfZPk01+/YapJkz1zSTX160dvk5a/VtiQOZE+kUyoE0W/SaLftUY6UZRrTeF/pW01PpT5ys4EbJ5", + "829LPS3KlF9HCbFJk9+kyb9p5XtKjnztXHlsB9Wy4y9Oer3ak+P9UOZNm2MjyZzVs+IvTnrprPh8Pf0L", + "8VZP58X158QngCw3Jz6ZtzgnHj2gcEJHbKy3mRe/6Mz0Q1Nm+tgOejMmp0sMf8XkdI3GVjo3PcULFAeM", + "yXhxqenqhLKZ6QWRKPX6grLEjfhSjyI0ZeilRncKyCKPQvHpbPqhVk3zTmjmDaV6a2RXG2/IqEczZHrH", + "WFk10VsD/0Wt1ZI1x91OW/204pGIfostTtdDVjgH3Ax1tVTw+DReLRO8HIJl20UxNOuRB74Q2i7PAo93", + "qDwJXL32ou6lWcpdF3qdR3zXrp5MIbbXSQpfE/piuJ5CdKdmxbpiDngMQ7UU8IWISuGoXyrp/clsg/Yr", + "2gabfqRvgV+VsI66tf4QEWrBAE9xiV4iQju97hIdomrG6u7QTq9b7Ai9RJDfhuer6fS6i3OGMjCW6wZl", + "MxY7QEOxcsvFvMTF2+wmWq9Jpuihkl9TIqrJk1nRmbowh2dMQyvt7tQoXbE29hNH64X5OuWkFV2d6owX", + "o83I0evRX3KDLdWbGRNDHifUjm/cl1Xdl2y33pDjMiGiusg8pcBUdlrGtF/VZZkA/iIzTLIbs69Sl9I8", + "V2VNvJVFcFfzV6qTeDV3ZSkAy7ZOFDBr4qysn57LXJUx1ZY7KuVbL/JTDv1QEez6kGk1qVyDZlFORq/j", + "h1wPymF4rGOxU6/GW9EJqSCo5oOsV/aZnY8LJqo3qLC3l6mwb3yKb4D3FDOCherjc9eWqMym2PezFZSY", + "xqTiqhLyRjyH6E3oAWtSZGJ9pHlZiYmXk9YLa0sUkRC4lpUeMAEQ7O9ZgwlFIISeE983RJ7tO8LFP0JP", + "0EE2HkO3CYIQDfETcoRb4isMcPDb1xa4ISgmoE9oIurLToDv6WQlWTUC2LP9MWNA6gK1GI2OMOH3sQt8", + "cDPdU5lG46aqF+uulWwKYGwKYLwlBltWX6JW5lqitqxgWYla+aAA71W44GxFJ6aBtak+seFoK8/Rckyi", + "VgVx2eUlamNEK8dyhMfjVVjOpt7Ept7Eclkn26C1uTVcyM+Yjpjc/3cEY1u+ilhbTYdS4z0I0QP2I6Ks", + "eKUcQI+hVuBCW5noYmNqsPFLCkm8HcN89kITb0pGbCpObCpOvDWFu6jIRO0OBILsENHiOMeliirA2GMM", + "XRcQ6ocMy8TXLXCJaBR6RP6g8UnhJfUj2vcYN4I2jfja+WucowvPM0F2FGI6AUEUBj5BRERb80GTKwnw", + "AqlOTFE13iD3II6/mGhvd3n4deOxc/dD/AdygJVtoxazrpVOrSXxGStMl6deHdGLYw9XDHWJVDEkIiLP", + "DicB70hGAVOYhMIin3ZPwTgilLu+uDrQ6nvssbRCifZ5RJhKRLmyg9my1DO2+XFH2AEa+iECAQoJJhR5", + "NjJhu3AkipUvKIVXDL6A60ilA9fkhZf6i6j/ITznHMAYn65iOhSedXFXQajYIl3+Z3mD4bhxJxVVpv0E", + "LqRDPxy3Hom/17L98c7DbqPZuMceO5b4QMaIQgdSvhfqHgakcAAJsgJIyKMfcjojAbLzaNjzCb0L0dU/", + "z8EYYg+oT0H8aTN1reO4care6OmDx6mFcgs6tHHc2GvvvbPau1b78Hq3fbzfPm63/8UUOscIY7Mhrczi", + "b5/5qb3g7MXpCpQW1pCJS4hPVyMO8gEmBq8Fxphw0vZDgKV2M8TIdcgKM/jXSgCXbDMJj3ZPVzLrG1g6", + "dxYqaVkwhyjKf4FU0nSuqZnfPRSOIVuoq+oSMLEldzfOAlf0zEQWJiI6PoKhIz/hx9D3PGb+2f4DCidg", + "jOwR9DAZCykXSx32LXbQOPDZiQBLjMCbsQLP9yx+dsijfU/CEEqt76B9YBJgIuVWE2B5fc1I/qasZrDl", + "+UDiyvZK09zBjKLL86klTJG08JJ74SPCrRW++br4ijPTG/I00tZWYuEkQoLN9Zs0e6rz86m7c1U+/6rQ", + "eixhGaVHISpKEK+DzJvl1hSRnW8580mIOqV1xtqlfE3XLvueSa20R0yRkMrlAIlcFUahyGmBrjDc1MuE", + "7wKgft+T43NmIuZuAggO2225c9xTJ4ZR3jlunmIbSBw0Ef9HREspfwYKUVclipQ7aXlB921pd/FiGiQK", + "9kOyb4f79L/WT+lTSO+U8I7EeNYIY31M6aX6sNaF3aJy1UrzLNXDcav48XP+qcQPLutIsn8+pVkNo1AS", + "8OhE91QjyyD0nZYzaDEKb6V4AhaO9RS/4r+lBzAwlOeaMvVKwuokFb7RlXWh5nLohCiK/0x5Ofpe4uaw", + "ozBkymKJu6MJkAcHrmzq748hZZID3wnM7XvUZ/OgUKShOlGYFGYnLfDFdTQXG2emzJKAAxeBBwylr0WX", + "gCZpJFb+5/SlzCpupVwoFLdxN4uNJ6W6UN09Pjh8BU/KSqQPTPWkCETaiPd1Eu/TPCcq5aE+r0k0iOFi", + "jMWrcDlH/wbwbwB8gNjl0qPKFZ0rbYAen3ORcafMZJUjULlVrm54xwDr4uunxF683OyAjiAFDhpiDxHA", + "I64uHmMqDHTImSagPI45lNlG+hik6NZH9igXpXNkplFlX17lvkMWmFImlzsIFcF5ReH0aj7z1b7HkCOa", + "mq9e5hn7zjf2n27Fuih5oq5aIcVApRkj0mCLCdBemJV/YHB+55Yh/eBL10A+r0chj0XiZUlJDx5zEQUj", + "eDaMAf/Ka328Hta1V4TXv1a9jc8rfzO3AJu412j5NTfysFSrvrFUDF+8VpW7MvC8spSlfDcbyjLboktU", + "ZaaYp6lXqxal7fS6TaBt5tRytFcpgGaqSds9BVtaidTuKZtLNFLcLiiJCgPMKbg0Vd38Ybyk+QYoKcba", + "Obnu/nzWaDa6n+N/Xp79/OXT2ekiSrJWpe15jPs1seuXYdLLrRxwgaVtAL+XXLkKS95YX4KhvjJGemXR", + "8me2zYGVlhrrVL6UpBF7YZJu55v+51x2+zwmeyW1Mg3Zgs3217LYU0B462e+r4LlXt1oXz7etV+X/7+W", + "vb5GaG0w3lfEbp/dZF8Kfi9Wx3o1k70yOr+Wpb5GNGU022vWYx7RYBD69yis0E3mFzT4wN+tp6XMFNM9", + "mY0BVtl0jz8rbyxzRX37HlyHor1M6qPF9ZhJw7bcbjNvrOH1TL1edFSq1vBlf6kNX1KEtdpXU9OgJrwo", + "jdoL6/+iT59tApN+KHMiCRhgB4fIFhwJEBoiyMtYDhB9RMhjX1359j2iwHYx2zme+vAJDu8hEKxRVroM", + "UGjZvueJsQAmvsvPo8itksK6xYh8fYp6ki3NIy7VRZOm1jy+po5505amqhcnTaFvqEGNjg91c6S8ilS9", + "X00KT6v6d3Tkr6Nrb3oXCvvYEKYOWVSoQ1ZqxWvQzaYc+mo9bVKn9WqNbaZDsWx7KQXRmrjWFskRSpvd", + "pDar3KFWG6Grtjep1a0bfc+gD9Sl3VQgv9fx+q0RxTGsz+G8syQhPH+TijRUVfLg9UXW0bMiJ5aN3Sve", + "DAmvSQeL9Kmscx+LnEfrxQT50q4WpTS3pr0tMlyhdqZgqqq5fJ6waXmxaXnxWvUqSzjy6/hRtpxITCII", + "0g/5prFHya3t7TXry7FQYTFNeVvBVh2LZOtLZd+z9epIgaaA2jTp2PDgNdKKc0xiOerwsnt4/Ok4VFxz", + "Y4kcatPTY9PT4zU5bVF3j43i+9LmIyum9dbXfWS6k2W1epD8CTXt+KTfhCzb9B7Z9B6pV7qtTTOSZcgP", + "Eg2q5eVeRYN6knL5vj/RKtm5cs6ZUnOvokF5Xu4vCNIRCrV3F5qOq+BZbi6uNrHc7x3J6pPZdx7FTljo", + "geF0ASDy88UkBWv5vG81LVjgcLWc4MNl5wQrAlv1hOCEEWgsUCH4IlOBxcTZPODiYJ087YXl4orxa0vE", + "zQ637CxcRRxGUS73fpN/O0P+raKJt5V8G1NVfdSfUX9myrmViDlDwm28gJcan2rVhWm2SqYna1uD7Foj", + "0JWTauVxvGZGbRkIr2AESXDWJ5d2AQQ+LYtW7tHUFFrxXl35s3JRa0K1VaV3LVrINNJ6tVTZtaAmmSer", + "YbVTt7Zc8Wp8AkW1e/ELEo8iXrBMQnujCn972Qr/SgXvVtLJuRYcqYw1LFyVf1HmvoKnYtq+WFJNOfsa", + "B5uesL+GasP65Omrk1j3JP3Eyf0ykqshPb+AsNY3Nz8m/Xopf2pW/nqpMZtk/E0y/svY7iYhqc5M/AVI", + "hFIdbDUT8BfAu5fBo1+Qca8g2mTcbxhtQvBrlCZTmHm/GB33FXLu3zhTMiTZL5wpbZLsN0n2K8dcNwpt", + "bRn2r6rN1ppYX+YeWbms+jevPhvT6NdWWm3S6Ddp9G/bOCjOoV+WhJBN9AtDTpeIRqFHgGrGLiB0XQBt", + "ih8Q+Omic6I68bdAz4WYp14L5k0ADBHwENsA7Nlu5CBnSkhKtoFeLwa94JyVke/fi32p3B5KNtPe8nwQ", + "qDPZ3kSkSvO2dVSunxCrpHGrWBRhtnk4Cah/F8JghG1+P4UgOwoVxY1gyISB6Aq/NfTDMaTH4OvjiCD7", + "t6/ge/DugOlLwB7BkGz3PZXPdYcfZHVnmdUlNLU03QpljRE+csB33/mejb77Tql4CtOZGtf3BHET6svL", + "NfFISUBLAgkJ/+ur+PMriOk79j+w533vqyTO0RjaFtvLryoUZox/PQr6APxCCCD4zoM0ChERWTSlMbAr", + "1YN+3VjNQrJ2Ei5Tb9RrytDLTt83wVLSF0PgrpLTafVL0E6M0EOMXGaOuC7ndjAIEGRyD0BvAoYRw0oZ", + "8g7BHaIxIbVe2wHx5wyZdRRjkoFzpN89SN03kMmwmAB5K2oFMypSSQyaJFusICtTKXe+iX9MjaD1UDiG", + "bFPcCQjR2H9ARJMbLXCmnA+K1TvIxQ8oxPw9SGU0jZ1PfKKuC/B4jBwMKXIn3CpJZANI7JWp9x/WUUaU", + "+gvkFsWXMVIA3WE6igaW3GgzMMmpLqBtngBuhe4kKItSbttaXU/gsGu84HVIv6K7kbsDdcKXamHiH3Qn", + "mhz2PSTEr+/GajChfkD6nqJu705TB3V2ID5kjDLRe3WNl+m7Bu028pDHtXLkmJRLg6fxDTOPAl/jUhlI", + "+9X1QpNTLsGuWDNUmJzWDOEdxF5rw9dmcpRtiS3fXhKTY2AwsxvTCadc/l0noqPG8a+3DCUF1CayPvdt", + "6AI5Gp+52YhCt3HcGFEaHO/suOyFkU/o8VH7qL0DA7wzjsHcedht5Gnx1LfvUbjzKRqg0EMUEa1WQXb4", + "OxGKsdgxhr7rorBwntt4z7ITnlzenIKYzwlNWDXYIwlJm3ru5aE3DXZx0uuF/hNG2mgXJz3AfpyUDyce", + "qiSG6/MrYKOQsU6bu1DY6D9dX/euQBSI/laAKY1DicfJdCfJV7PDf35+wWB9wA4KwTUaBy4bJuUf11Zm", + "fvtlk1aaa94pnibTxp92SqbBEyevHEv+kBrp9vn/BQAA//9BwurpDl8CAA==", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/gateway/gateway-controller/pkg/config/api_validator.go b/gateway/gateway-controller/pkg/config/api_validator.go index bb0698f66a..daefcfee0a 100644 --- a/gateway/gateway-controller/pkg/config/api_validator.go +++ b/gateway/gateway-controller/pkg/config/api_validator.go @@ -26,6 +26,7 @@ import ( "time" api "github.com/wso2/api-platform/gateway/gateway-controller/pkg/api/management" + "github.com/wso2/api-platform/gateway/gateway-controller/pkg/constants" ) // APIValidator validates API configurations using rule-based validation @@ -436,12 +437,63 @@ func (v *APIValidator) validateRestData(spec *api.APIConfigData) []ValidationErr errors = append(errors, v.validateUpstream("sandbox", spec.Upstream.Sandbox, spec.UpstreamDefinitions)...) } + // Validate API-level resilience block + errors = append(errors, v.validateResilience("spec.resilience", spec.Resilience)...) + // Validate operations errors = append(errors, v.validateOperations(spec.Operations)...) return errors } +// validateResilience validates a resilience block (timeout / idleTimeout). Both fields +// are optional duration strings; "0s" is allowed (disables the timeout), negative and +// malformed values are rejected. fieldPrefix is the path to the block (e.g. +// "spec.resilience" or "spec.operations[2].resilience"). +func (v *APIValidator) validateResilience(fieldPrefix string, r *api.Resilience) []ValidationError { + return validateResilienceTimeouts(fieldPrefix, r) +} + +// validateResilienceTimeouts validates the timeout fields of a resilience block. +func validateResilienceTimeouts(fieldPrefix string, r *api.Resilience) []ValidationError { + var errors []ValidationError + if r == nil { + return errors + } + + validate := func(field string, value *string) { + if value == nil { + return + } + s := strings.TrimSpace(*value) + if s == "" { + return + } + // Enforce the same single-unit format as the CRD admission controller (see + // constants.ResilienceDurationPattern). This rejects compound durations ("1h30m"), + // negatives ("-30s"), and unitless values ("0", "30"), while accepting "0s" to disable. + if !constants.ResilienceDurationRegex.MatchString(s) { + errors = append(errors, ValidationError{ + Field: field, + Message: "Invalid timeout format (expected a single-unit duration like '30s', '1m', '500ms', or '0s' to disable; compound, negative, and unitless values are not allowed)", + }) + return + } + // The pattern guarantees a parseable, non-negative, single-unit value; ParseDuration is a + // final guard against pathological overflow. + if _, err := time.ParseDuration(s); err != nil { + errors = append(errors, ValidationError{ + Field: field, + Message: fmt.Sprintf("Invalid timeout format: %v", err), + }) + } + } + + validate(fieldPrefix+".timeout", r.Timeout) + validate(fieldPrefix+".idleTimeout", r.IdleTimeout) + return errors +} + // validateAsyncData validates the data section of the configuration for http/rest kind func (v *APIValidator) validateAsyncData(spec *api.WebhookAPIData) []ValidationError { var errors []ValidationError @@ -621,6 +673,9 @@ func (v *APIValidator) validateOperations(operations []api.Operation) []Validati Message: "Operation path has unbalanced braces in parameters", }) } + + // Validate operation-level resilience block + errors = append(errors, v.validateResilience(fmt.Sprintf("spec.operations[%d].resilience", i), op.Resilience)...) } return errors diff --git a/gateway/gateway-controller/pkg/config/api_validator_test.go b/gateway/gateway-controller/pkg/config/api_validator_test.go index f93a0d58d4..8745ee6cdf 100644 --- a/gateway/gateway-controller/pkg/config/api_validator_test.go +++ b/gateway/gateway-controller/pkg/config/api_validator_test.go @@ -478,6 +478,60 @@ func TestAPIValidator_ValidateOperations(t *testing.T) { } } +func TestAPIValidator_ValidateResilience(t *testing.T) { + v := NewAPIValidator() + + tests := []struct { + name string + apiRes *api.Resilience + opRes *api.Resilience + wantError bool + errField string + }{ + {name: "No resilience is valid", wantError: false}, + {name: "Valid API-level timeout + idleTimeout", apiRes: &api.Resilience{Timeout: stringPtr("15s"), IdleTimeout: stringPtr("0s")}, wantError: false}, + {name: "Valid operation-level timeout", opRes: &api.Resilience{Timeout: stringPtr("2s")}, wantError: false}, + {name: "0s is allowed (disabled)", apiRes: &api.Resilience{Timeout: stringPtr("0s"), IdleTimeout: stringPtr("0s")}, wantError: false}, + {name: "Invalid API-level timeout format", apiRes: &api.Resilience{Timeout: stringPtr("15seconds")}, wantError: true, errField: "spec.resilience.timeout"}, + {name: "Invalid API-level idleTimeout format", apiRes: &api.Resilience{IdleTimeout: stringPtr("abc")}, wantError: true, errField: "spec.resilience.idleTimeout"}, + {name: "Negative API-level timeout rejected", apiRes: &api.Resilience{Timeout: stringPtr("-5s")}, wantError: true, errField: "spec.resilience.timeout"}, + {name: "Compound API-level timeout rejected (must match CRD pattern)", apiRes: &api.Resilience{Timeout: stringPtr("1h30m")}, wantError: true, errField: "spec.resilience.timeout"}, + {name: "Compound API-level idleTimeout rejected", apiRes: &api.Resilience{IdleTimeout: stringPtr("1m30s")}, wantError: true, errField: "spec.resilience.idleTimeout"}, + {name: "Unitless API-level timeout rejected", apiRes: &api.Resilience{Timeout: stringPtr("30")}, wantError: true, errField: "spec.resilience.timeout"}, + {name: "Bare 0 rejected (unit required)", apiRes: &api.Resilience{Timeout: stringPtr("0")}, wantError: true, errField: "spec.resilience.timeout"}, + {name: "Fractional single-unit timeout allowed", apiRes: &api.Resilience{Timeout: stringPtr("1.5s")}, wantError: false}, + {name: "Invalid operation-level timeout format", opRes: &api.Resilience{Timeout: stringPtr("nope")}, wantError: true, errField: "spec.operations[0].resilience.timeout"}, + {name: "Compound operation-level timeout rejected", opRes: &api.Resilience{Timeout: stringPtr("1h30m")}, wantError: true, errField: "spec.operations[0].resilience.timeout"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := createValidRestAPIConfig() + cfg.Spec.Resilience = tt.apiRes + cfg.Spec.Operations[0].Resilience = tt.opRes + + errors := v.Validate(cfg) + hasExpectedError := false + for _, e := range errors { + if e.Field == tt.errField { + hasExpectedError = true + break + } + } + if tt.wantError && !hasExpectedError { + t.Errorf("expected error for field %s, got: %v", tt.errField, errors) + } + if !tt.wantError { + for _, e := range errors { + if strings.Contains(e.Field, "resilience") { + t.Errorf("unexpected resilience error: %v", e) + } + } + } + }) + } +} + func TestAPIValidator_ValidateAllHTTPMethods(t *testing.T) { v := NewAPIValidator() diff --git a/gateway/gateway-controller/pkg/config/config.go b/gateway/gateway-controller/pkg/config/config.go index 6402427984..83f9060974 100644 --- a/gateway/gateway-controller/pkg/config/config.go +++ b/gateway/gateway-controller/pkg/config/config.go @@ -520,8 +520,17 @@ type VHostEntry struct { // HTTPListenerConfig holds HTTP listener related configuration of an API type HTTPListenerConfig struct { - ServerHeaderTransformation string `koanf:"server_header_transformation"` // Options: "APPEND_IF_ABSENT", "OVERWRITE", "PASS_THROUGH" - ServerHeaderValue string `koanf:"server_header_value"` // Custom value for the Server header + ServerHeaderTransformation string `koanf:"server_header_transformation"` // Options: "APPEND_IF_ABSENT", "OVERWRITE", "PASS_THROUGH" + ServerHeaderValue string `koanf:"server_header_value"` // Custom value for the Server header + Timeouts HCMTimeouts `koanf:"timeouts"` // HTTP Connection Manager (downstream) timeouts +} + +// HCMTimeouts holds HTTP Connection Manager (downstream/connection) timeouts. +type HCMTimeouts struct { + RequestTimeout time.Duration `koanf:"request_timeout"` // HCM request_timeout (default 0s = disabled) + RequestHeadersTimeout time.Duration `koanf:"request_headers_timeout"` // HCM request_headers_timeout (default 0s = disabled) + StreamIdleTimeout time.Duration `koanf:"stream_idle_timeout"` // HCM stream_idle_timeout (default 5m) + IdleTimeout time.Duration `koanf:"idle_timeout"` // common_http_protocol_options.idle_timeout (default 1h) } // PolicyEngineConfig holds policy engine ext_proc filter configuration @@ -893,6 +902,12 @@ func defaultConfig() *Config { HTTPListener: HTTPListenerConfig{ ServerHeaderTransformation: commonconstants.OVERWRITE, ServerHeaderValue: commonconstants.ServerName, + Timeouts: HCMTimeouts{ + RequestTimeout: 0, // 0s = disabled (Envoy default) + RequestHeadersTimeout: 0, // 0s = disabled (Envoy default) + StreamIdleTimeout: 5 * time.Minute, // Envoy default + IdleTimeout: 1 * time.Hour, // Envoy default (connection-level) + }, }, }, Analytics: AnalyticsConfig{ @@ -1592,6 +1607,26 @@ func (c *Config) validateTimeoutConfig() error { timeouts.ConnectTimeoutMs, constants.MaxReasonableTimeoutMs) } + // Validate HCM (downstream/connection) timeouts. Unlike the upstream timeouts above, + // 0 is a valid value here: which denotes "disabled scenario" + // only reject negative values and unreasonably large ones are rejected. + maxConnTimeout := time.Duration(constants.MaxReasonableConnectionTimeoutMs) * time.Millisecond + hcmTimeouts := map[string]time.Duration{ + "request_timeout": c.Router.HTTPListener.Timeouts.RequestTimeout, + "request_headers_timeout": c.Router.HTTPListener.Timeouts.RequestHeadersTimeout, + "stream_idle_timeout": c.Router.HTTPListener.Timeouts.StreamIdleTimeout, + "idle_timeout": c.Router.HTTPListener.Timeouts.IdleTimeout, + } + for name, v := range hcmTimeouts { + if v < 0 { + return fmt.Errorf("router.http_listener.timeouts.%s must not be negative, got: %s", name, v) + } + if v > maxConnTimeout { + return fmt.Errorf("router.http_listener.timeouts.%s (%s) exceeds maximum reasonable timeout of %s", + name, v, maxConnTimeout) + } + } + return nil } diff --git a/gateway/gateway-controller/pkg/config/config_test.go b/gateway/gateway-controller/pkg/config/config_test.go index 91e53733a4..5d6538a15d 100644 --- a/gateway/gateway-controller/pkg/config/config_test.go +++ b/gateway/gateway-controller/pkg/config/config_test.go @@ -1004,6 +1004,61 @@ func TestConfig_ValidateTimeoutConfig(t *testing.T) { } } +func TestConfig_ValidateHCMTimeouts(t *testing.T) { + maxConn := time.Duration(constants.MaxReasonableConnectionTimeoutMs) * time.Millisecond + tests := []struct { + name string + timeouts HCMTimeouts + wantErr bool + errContains string + }{ + { + name: "Envoy defaults are valid", + timeouts: HCMTimeouts{RequestTimeout: 0, RequestHeadersTimeout: 0, StreamIdleTimeout: 5 * time.Minute, IdleTimeout: time.Hour}, + }, + { + name: "All zero (disabled) is valid", + timeouts: HCMTimeouts{}, + }, + { + name: "Custom positive values are valid", + timeouts: HCMTimeouts{RequestTimeout: 30 * time.Second, RequestHeadersTimeout: 10 * time.Second, StreamIdleTimeout: time.Minute, IdleTimeout: 2 * time.Hour}, + }, + { + name: "Negative request_timeout rejected", + timeouts: HCMTimeouts{RequestTimeout: -1 * time.Second}, + wantErr: true, + errContains: "router.http_listener.timeouts.request_timeout must not be negative", + }, + { + name: "Negative stream_idle_timeout rejected", + timeouts: HCMTimeouts{StreamIdleTimeout: -5 * time.Second}, + wantErr: true, + errContains: "router.http_listener.timeouts.stream_idle_timeout must not be negative", + }, + { + name: "idle_timeout exceeding max reasonable rejected", + timeouts: HCMTimeouts{IdleTimeout: maxConn + time.Second}, + wantErr: true, + errContains: "router.http_listener.timeouts.idle_timeout", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := validConfig() + cfg.Router.HTTPListener.Timeouts = tt.timeouts + err := cfg.Validate() + if tt.wantErr { + assert.Error(t, err) + assert.Contains(t, err.Error(), tt.errContains) + } else { + assert.NoError(t, err) + } + }) + } +} + func TestConfig_ValidatePolicyEngineConfig(t *testing.T) { tests := []struct { name string @@ -1582,6 +1637,50 @@ func TestDefaultConfig(t *testing.T) { assert.Equal(t, "info", cfg.Controller.Logging.Level) assert.False(t, cfg.Controller.Server.SkipInvalidDeploymentsOnStartup) assert.Equal(t, uint32(5000), cfg.Router.Upstream.Timeouts.ConnectTimeoutMs, "default router.upstream.timeouts.connect_timeout_ms should be 5s (5000 ms)") + + // HCM timeout defaults must match Envoy's documented defaults. + hcm := cfg.Router.HTTPListener.Timeouts + assert.Equal(t, time.Duration(0), hcm.RequestTimeout, "default request_timeout should be 0s (disabled)") + assert.Equal(t, time.Duration(0), hcm.RequestHeadersTimeout, "default request_headers_timeout should be 0s (disabled)") + assert.Equal(t, 5*time.Minute, hcm.StreamIdleTimeout, "default stream_idle_timeout should be 5m") + assert.Equal(t, time.Hour, hcm.IdleTimeout, "default idle_timeout should be 1h") +} + +func TestLoadConfig_HCMTimeouts(t *testing.T) { + t.Run("explicit values parse from toml", func(t *testing.T) { + tmpDir := t.TempDir() + configPath := filepath.Join(tmpDir, "config.toml") + toml := ` +[router.http_listener.timeouts] +request_timeout = "30s" +request_headers_timeout = "10s" +stream_idle_timeout = "2m" +idle_timeout = "30m" +` + require.NoError(t, os.WriteFile(configPath, []byte(toml), 0o644)) + + cfg, err := LoadConfig(configPath) + require.NoError(t, err) + hcm := cfg.Router.HTTPListener.Timeouts + assert.Equal(t, 30*time.Second, hcm.RequestTimeout) + assert.Equal(t, 10*time.Second, hcm.RequestHeadersTimeout) + assert.Equal(t, 2*time.Minute, hcm.StreamIdleTimeout) + assert.Equal(t, 30*time.Minute, hcm.IdleTimeout) + }) + + t.Run("omitted section falls back to Envoy defaults", func(t *testing.T) { + tmpDir := t.TempDir() + configPath := filepath.Join(tmpDir, "config.toml") + require.NoError(t, os.WriteFile(configPath, []byte(""), 0o644)) + + cfg, err := LoadConfig(configPath) + require.NoError(t, err) + hcm := cfg.Router.HTTPListener.Timeouts + assert.Equal(t, time.Duration(0), hcm.RequestTimeout) + assert.Equal(t, time.Duration(0), hcm.RequestHeadersTimeout) + assert.Equal(t, 5*time.Minute, hcm.StreamIdleTimeout) + assert.Equal(t, time.Hour, hcm.IdleTimeout) + }) } func TestConfig_CaseInsensitiveAlgorithm(t *testing.T) { diff --git a/gateway/gateway-controller/pkg/config/llm_validator.go b/gateway/gateway-controller/pkg/config/llm_validator.go index 9a1ca3cf08..67c82965ff 100644 --- a/gateway/gateway-controller/pkg/config/llm_validator.go +++ b/gateway/gateway-controller/pkg/config/llm_validator.go @@ -387,6 +387,10 @@ func (v *LLMValidator) validateProviderSpec(spec *api.LLMProviderConfigData) []V // The deprecated `policies` list must not coexist with the new policy lists errors = append(errors, v.validatePolicyListExclusivity(spec.GlobalPolicies, spec.OperationPolicies, spec.Policies)...) + // Validate API-level resilience (timeout / idleTimeout). LLM kinds support resilience at + // the API level only. + errors = append(errors, validateResilienceTimeouts("spec.resilience", spec.Resilience)...) + return errors } @@ -594,6 +598,10 @@ func (v *LLMValidator) validateProxyData(spec *api.LLMProxyConfigData) []Validat // The deprecated `policies` list must not coexist with the new policy lists errors = append(errors, v.validatePolicyListExclusivity(spec.GlobalPolicies, spec.OperationPolicies, spec.Policies)...) + // Validate API-level resilience (timeout / idleTimeout). LLM kinds support resilience at + // the API level only. + errors = append(errors, validateResilienceTimeouts("spec.resilience", spec.Resilience)...) + return errors } diff --git a/gateway/gateway-controller/pkg/config/llm_validator_resilience_test.go b/gateway/gateway-controller/pkg/config/llm_validator_resilience_test.go new file mode 100644 index 0000000000..db67f81e7f --- /dev/null +++ b/gateway/gateway-controller/pkg/config/llm_validator_resilience_test.go @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2026, WSO2 LLC. (https://www.wso2.com). + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package config + +import ( + "testing" + + "github.com/stretchr/testify/assert" + api "github.com/wso2/api-platform/gateway/gateway-controller/pkg/api/management" +) + +func validProviderWithResilience(r *api.Resilience) api.LLMProviderConfiguration { + return api.LLMProviderConfiguration{ + ApiVersion: "gateway.api-platform.wso2.com/v1", + Kind: api.LLMProviderConfigurationKindLlmProvider, + Metadata: api.Metadata{Name: "openai"}, + Spec: api.LLMProviderConfigData{ + DisplayName: "my-provider", + Version: "v1.0", + Template: "openai", + Upstream: api.LLMProviderConfigData_Upstream{Url: stringPtr("https://api.openai.com")}, + AccessControl: api.LLMAccessControl{Mode: api.AllowAll}, + Resilience: r, + }, + } +} + +func validProxyWithResilience(r *api.Resilience) api.LLMProxyConfiguration { + return api.LLMProxyConfiguration{ + ApiVersion: api.LLMProxyConfigurationApiVersionGatewayApiPlatformWso2Comv1, + Kind: api.LLMProxyConfigurationKindLlmProxy, + Metadata: api.Metadata{Name: "openai-proxy"}, + Spec: api.LLMProxyConfigData{ + DisplayName: "my-proxy", + Version: "v1.0", + Provider: api.LLMProxyProvider{Id: "openai"}, + Resilience: r, + }, + } +} + +func TestValidateLLMProvider_Resilience(t *testing.T) { + validator := NewLLMValidator() + + t.Run("valid timeout and idleTimeout", func(t *testing.T) { + errs := validator.Validate(validProviderWithResilience(&api.Resilience{ + Timeout: stringPtr("30s"), + IdleTimeout: stringPtr("0s"), + })) + assert.Empty(t, errs) + }) + + t.Run("nil resilience is fine", func(t *testing.T) { + errs := validator.Validate(validProviderWithResilience(nil)) + assert.Empty(t, errs) + }) + + t.Run("malformed timeout is rejected", func(t *testing.T) { + errs := validator.Validate(validProviderWithResilience(&api.Resilience{Timeout: stringPtr("30")})) + assertHasFieldError(t, errs, "spec.resilience.timeout") + }) + + t.Run("compound timeout is rejected (must match CRD pattern)", func(t *testing.T) { + errs := validator.Validate(validProviderWithResilience(&api.Resilience{Timeout: stringPtr("1h30m")})) + assertHasFieldError(t, errs, "spec.resilience.timeout") + }) + + t.Run("0s is accepted (disables)", func(t *testing.T) { + errs := validator.Validate(validProviderWithResilience(&api.Resilience{Timeout: stringPtr("0s")})) + assert.Empty(t, errs) + }) + + t.Run("negative timeout is rejected", func(t *testing.T) { + errs := validator.Validate(validProviderWithResilience(&api.Resilience{Timeout: stringPtr("-5s")})) + assertHasFieldError(t, errs, "spec.resilience.timeout") + }) + + t.Run("malformed idleTimeout is rejected", func(t *testing.T) { + errs := validator.Validate(validProviderWithResilience(&api.Resilience{IdleTimeout: stringPtr("abc")})) + assertHasFieldError(t, errs, "spec.resilience.idleTimeout") + }) +} + +func TestValidateLLMProxy_Resilience(t *testing.T) { + validator := NewLLMValidator() + + t.Run("valid timeout", func(t *testing.T) { + errs := validator.Validate(validProxyWithResilience(&api.Resilience{Timeout: stringPtr("75s")})) + assert.Empty(t, errs) + }) + + t.Run("nil resilience is fine", func(t *testing.T) { + errs := validator.Validate(validProxyWithResilience(nil)) + assert.Empty(t, errs) + }) + + t.Run("malformed timeout is rejected", func(t *testing.T) { + errs := validator.Validate(validProxyWithResilience(&api.Resilience{Timeout: stringPtr("fast")})) + assertHasFieldError(t, errs, "spec.resilience.timeout") + }) + + t.Run("negative idleTimeout is rejected", func(t *testing.T) { + errs := validator.Validate(validProxyWithResilience(&api.Resilience{IdleTimeout: stringPtr("-1s")})) + assertHasFieldError(t, errs, "spec.resilience.idleTimeout") + }) +} + +func assertHasFieldError(t *testing.T, errs []ValidationError, field string) { + t.Helper() + for _, e := range errs { + if e.Field == field { + return + } + } + t.Fatalf("expected a validation error on field %q, got %+v", field, errs) +} diff --git a/gateway/gateway-controller/pkg/constants/constants.go b/gateway/gateway-controller/pkg/constants/constants.go index b0741044af..4833f0502d 100644 --- a/gateway/gateway-controller/pkg/constants/constants.go +++ b/gateway/gateway-controller/pkg/constants/constants.go @@ -71,6 +71,10 @@ const ( // Configuration Validation Constants MaxReasonableTimeoutMs = uint32(3600000) // 1 hour in milliseconds MaxReasonablePolicyTimeoutMs = uint32(60000) // 60 seconds in milliseconds + + // MaxReasonableConnectionTimeoutMs caps connection-level timeouts (request, request-headers,etc.), + // allowing higher values than MaxReasonableTimeoutMs to support long-lived idle connections. + MaxReasonableConnectionTimeoutMs = uint32(86400000) // 24 hours in milliseconds // Cipher Suite Validation CipherInvalidChars1 = ";" @@ -175,6 +179,12 @@ const ( // System policy constants ANALYTICS_SYSTEM_POLICY_NAME = "wso2_apip_sys_analytics" ANALYTICS_SYSTEM_POLICY_VERSION = "v1" + + // ResilienceDurationPattern is the single source of truth for the format of resilience + // timeout strings. + // - accepted: "30s", "500ms", "1m", "2h", "1.5s", and "0s" (zero disables the timeout) + // - rejected: compound durations ("1h30m"), negatives ("-30s"), and unitless values ("0", "30") + ResilienceDurationPattern = `^\d+(\.\d+)?(ms|s|m|h)$` ) // DP->CP artifact push timing. The bottom-up (DP->CP) push waits for the local deployment @@ -195,3 +205,8 @@ var WILDCARD_HTTP_METHODS = []string{ "DELETE", "OPTIONS", } + +// ResilienceDurationRegex is the compiled ResilienceDurationPattern, shared by the management-API +// validator and the xDS downstream parser so both enforce exactly what the +// CRD admission controller enforces. +var ResilienceDurationRegex = regexp.MustCompile(ResilienceDurationPattern) \ No newline at end of file diff --git a/gateway/gateway-controller/pkg/models/runtime_deploy_config.go b/gateway/gateway-controller/pkg/models/runtime_deploy_config.go index 8dec6fa8c7..e5111228ec 100644 --- a/gateway/gateway-controller/pkg/models/runtime_deploy_config.go +++ b/gateway/gateway-controller/pkg/models/runtime_deploy_config.go @@ -62,8 +62,13 @@ type Route struct { } // RouteTimeout holds parsed timeout values for a route. +// Timeout and IdleTimeout come from the resilience block (operation-level overriding +// API-level). A nil field means "not configured" — the global route timeout default +// applies. A non-nil zero value means "explicitly disabled". type RouteTimeout struct { - Connect *time.Duration + Connect *time.Duration + Timeout *time.Duration // route timeout -> RouteAction.Timeout + IdleTimeout *time.Duration // route idle timeout -> RouteAction.IdleTimeout } // RouteUpstream links a route to its upstream cluster. diff --git a/gateway/gateway-controller/pkg/transform/restapi.go b/gateway/gateway-controller/pkg/transform/restapi.go index 8d404dca3d..053fdb3223 100644 --- a/gateway/gateway-controller/pkg/transform/restapi.go +++ b/gateway/gateway-controller/pkg/transform/restapi.go @@ -24,6 +24,7 @@ import ( "net/url" "strconv" "strings" + "time" commonconstants "github.com/wso2/api-platform/common/constants" versionutil "github.com/wso2/api-platform/common/version" @@ -151,8 +152,22 @@ func (t *RestAPITransformer) Transform(cfg *models.StoredConfig) (*models.Runtim return nil, fmt.Errorf("sandbox upstream is configured but resolves to the same vhost %q as the main upstream; configure distinct vhosts to avoid route conflicts", effectiveMainVHost) } + // Resolve API-level resilience timeouts once; operation-level values override these. + apiTimeout, apiIdleTimeout, err := xds.ResolveResilience(apiData.Resilience) + if err != nil { + return nil, fmt.Errorf("invalid API-level resilience: %w", err) + } + // Build routes and policy chains for each operation for _, op := range apiData.Operations { + // Operation-level resilience overrides API-level (per field); nil leaves the + // global route timeout default in effect. + opTimeout, opIdleTimeout, err := xds.ResolveResilience(op.Resilience) + if err != nil { + return nil, fmt.Errorf("invalid resilience for operation %s %s: %w", op.Method, op.Path, err) + } + routeTimeout := buildRouteTimeout(opTimeout, apiTimeout, opIdleTimeout, apiIdleTimeout) + vhosts := []string{effectiveMainVHost} if hasSandbox { vhosts = append(vhosts, effectiveSandboxVHost) @@ -168,6 +183,7 @@ func (t *RestAPITransformer) Transform(cfg *models.StoredConfig) (*models.Runtim OperationPath: op.Path, Vhost: vhost, AutoHostRewrite: mainAutoHostRewrite, + Timeout: routeTimeout, Upstream: models.RouteUpstream{ ClusterKey: mainUpstream.ClusterKey, UseClusterHeader: useClusterHeader, @@ -246,6 +262,24 @@ func (t *RestAPITransformer) Transform(cfg *models.StoredConfig) (*models.Runtim } // collectAPIPolicies validates and collects API-level policies into SDK format. +// buildRouteTimeout applies operation-over-API precedence (per field) and returns a +// *models.RouteTimeout, or nil when neither level configured any timeout (so the global +// route timeout default applies). +func buildRouteTimeout(opTimeout, apiTimeout, opIdle, apiIdle *time.Duration) *models.RouteTimeout { + timeout := opTimeout + if timeout == nil { + timeout = apiTimeout + } + idle := opIdle + if idle == nil { + idle = apiIdle + } + if timeout == nil && idle == nil { + return nil + } + return &models.RouteTimeout{Timeout: timeout, IdleTimeout: idle} +} + func (t *RestAPITransformer) collectAPIPolicies(policies *[]api.Policy) map[string]policyenginev1.PolicyInstance { result := make(map[string]policyenginev1.PolicyInstance) if policies == nil { diff --git a/gateway/gateway-controller/pkg/transform/restapi_test.go b/gateway/gateway-controller/pkg/transform/restapi_test.go index 5edf451df9..2f5835715c 100644 --- a/gateway/gateway-controller/pkg/transform/restapi_test.go +++ b/gateway/gateway-controller/pkg/transform/restapi_test.go @@ -21,6 +21,7 @@ package transform import ( "net/url" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -84,6 +85,79 @@ func makeRestAPIStoredConfig(apiPolicies []api.Policy, opPolicies []api.Policy) } } +// makeRestAPIStoredConfigWithResilience builds a RestAPI StoredConfig whose single +// operation (GET /hello) carries optional API-level and operation-level resilience blocks. +func makeRestAPIStoredConfigWithResilience(apiRes, opRes *api.Resilience) *models.StoredConfig { + apiData := api.APIConfigData{ + DisplayName: "Test API", + Context: "/test", + Version: "1.0.0", + Resilience: apiRes, + Operations: []api.Operation{ + {Method: "GET", Path: "/hello", Resilience: opRes}, + }, + Upstream: struct { + Main api.Upstream `json:"main" yaml:"main"` + Sandbox *api.Upstream `json:"sandbox,omitempty" yaml:"sandbox,omitempty"` + }{ + Main: api.Upstream{Url: ptrStr("http://backend:8080")}, + }, + } + return &models.StoredConfig{ + UUID: "test-api", + Kind: string(api.RestAPIKindRestApi), + Configuration: api.RestAPI{Kind: api.RestAPIKindRestApi, Metadata: api.Metadata{Name: "test-api"}, Spec: apiData}, + } +} + +func TestRestAPITransformer_ResiliencePrecedence(t *testing.T) { + const routeKey = "GET|/test/hello|main.local" + transformer := NewRestAPITransformer(testRouterCfg(), &config.Config{}, map[string]models.PolicyDefinition{}) + + t.Run("operation-level overrides API-level", func(t *testing.T) { + cfg := makeRestAPIStoredConfigWithResilience( + &api.Resilience{Timeout: ptrStr("10s"), IdleTimeout: ptrStr("30s")}, + &api.Resilience{Timeout: ptrStr("2s")}, + ) + rdc, err := transformer.Transform(cfg) + require.NoError(t, err) + rt := rdc.Routes[routeKey].Timeout + require.NotNil(t, rt) + require.NotNil(t, rt.Timeout) + assert.Equal(t, 2*time.Second, *rt.Timeout, "operation timeout should win") + require.NotNil(t, rt.IdleTimeout) + assert.Equal(t, 30*time.Second, *rt.IdleTimeout, "idleTimeout falls back to API-level when op omits it") + }) + + t.Run("API-level applies when operation omits resilience", func(t *testing.T) { + cfg := makeRestAPIStoredConfigWithResilience(&api.Resilience{Timeout: ptrStr("7s")}, nil) + rdc, err := transformer.Transform(cfg) + require.NoError(t, err) + rt := rdc.Routes[routeKey].Timeout + require.NotNil(t, rt) + require.NotNil(t, rt.Timeout) + assert.Equal(t, 7*time.Second, *rt.Timeout) + assert.Nil(t, rt.IdleTimeout) + }) + + t.Run("no resilience leaves Timeout nil (global default applies)", func(t *testing.T) { + cfg := makeRestAPIStoredConfigWithResilience(nil, nil) + rdc, err := transformer.Transform(cfg) + require.NoError(t, err) + assert.Nil(t, rdc.Routes[routeKey].Timeout) + }) + + t.Run("0s is preserved as explicit disable", func(t *testing.T) { + cfg := makeRestAPIStoredConfigWithResilience(&api.Resilience{Timeout: ptrStr("0s")}, nil) + rdc, err := transformer.Transform(cfg) + require.NoError(t, err) + rt := rdc.Routes[routeKey].Timeout + require.NotNil(t, rt) + require.NotNil(t, rt.Timeout) + assert.Equal(t, time.Duration(0), *rt.Timeout) + }) +} + // findPolicyInChain returns true if any policy chain for the given route key // contains a policy with the given name. func findPolicyInChain(rdc *models.RuntimeDeployConfig, routeKey, policyName string) bool { diff --git a/gateway/gateway-controller/pkg/utils/llm_resilience_test.go b/gateway/gateway-controller/pkg/utils/llm_resilience_test.go new file mode 100644 index 0000000000..38cb518f17 --- /dev/null +++ b/gateway/gateway-controller/pkg/utils/llm_resilience_test.go @@ -0,0 +1,236 @@ +/* + * Copyright (c) 2026, WSO2 LLC. (https://www.wso2.com). + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package utils + +import ( + "io" + "log/slog" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + api "github.com/wso2/api-platform/gateway/gateway-controller/pkg/api/management" + "github.com/wso2/api-platform/gateway/gateway-controller/pkg/config" + "github.com/wso2/api-platform/gateway/gateway-controller/pkg/models" + "github.com/wso2/api-platform/gateway/gateway-controller/pkg/storage" +) + +// These tests pin the LLM resilience route-mapping rule: API-level resilience is attached to every +// traffic-forwarding route and never to the access-control deny routes. + +// In allow_all mode, the catch-all (and any policy-derived routes) carry the resilience block while +// the exception/deny routes (which only return a 404) are left untouched. +func TestTransformProvider_AllowAll_ResilienceSkipsDenyRoutes(t *testing.T) { + transformer, _ := setupTestTransformer(t) + + exceptions := []api.RouteException{ + {Path: "/admin", Methods: []api.RouteExceptionMethods{api.RouteExceptionMethodsGET, api.RouteExceptionMethodsPOST}}, + } + timeout := "30s" + provider := &api.LLMProviderConfiguration{ + ApiVersion: "gateway.api-platform.wso2.com/v1", + Kind: "LlmProvider", + Metadata: api.Metadata{Name: "openai-provider"}, + Spec: api.LLMProviderConfigData{ + DisplayName: "test", + Version: "v1.0", + Template: "openai", + Upstream: api.LLMProviderConfigData_Upstream{Url: stringPtr("https://api.example.com")}, + AccessControl: api.LLMAccessControl{ + Mode: api.AllowAll, + Exceptions: &exceptions, + }, + Resilience: &api.Resilience{Timeout: &timeout}, + }, + } + + result, err := transformer.Transform(provider, &api.RestAPI{}) + require.NoError(t, err) + + sawCatchAll := false + sawDeny := false + for _, op := range result.Spec.Operations { + if op.Path == "/admin" { // the deny/exception routes + sawDeny = true + assert.Nil(t, op.Resilience, "deny route %s %s must not carry resilience", op.Method, op.Path) + continue + } + // catch-all (traffic-forwarding) routes + sawCatchAll = true + require.NotNil(t, op.Resilience, "traffic route %s %s should carry resilience", op.Method, op.Path) + require.NotNil(t, op.Resilience.Timeout) + assert.Equal(t, "30s", *op.Resilience.Timeout) + } + assert.True(t, sawCatchAll, "expected at least one traffic-forwarding route") + assert.True(t, sawDeny, "expected at least one deny route") +} + +// In deny_all mode every created route is an allow-listed forwarding route, so all of them carry the +// resilience block. +func TestTransformProvider_DenyAll_ResilienceOnAllRoutes(t *testing.T) { + transformer, _ := setupTestTransformer(t) + + exceptions := []api.RouteException{ + {Path: "/chat/completions", Methods: []api.RouteExceptionMethods{api.RouteExceptionMethodsPOST}}, + {Path: "/models", Methods: []api.RouteExceptionMethods{api.RouteExceptionMethodsGET}}, + } + timeout := "45s" + idle := "0s" + provider := &api.LLMProviderConfiguration{ + ApiVersion: "gateway.api-platform.wso2.com/v1", + Kind: "LlmProvider", + Metadata: api.Metadata{Name: "openai-provider"}, + Spec: api.LLMProviderConfigData{ + DisplayName: "test", + Version: "v1.0", + Template: "openai", + Upstream: api.LLMProviderConfigData_Upstream{Url: stringPtr("https://api.example.com")}, + AccessControl: api.LLMAccessControl{ + Mode: api.DenyAll, + Exceptions: &exceptions, + }, + Resilience: &api.Resilience{Timeout: &timeout, IdleTimeout: &idle}, + }, + } + + result, err := transformer.Transform(provider, &api.RestAPI{}) + require.NoError(t, err) + require.NotEmpty(t, result.Spec.Operations) + + for _, op := range result.Spec.Operations { + require.NotNil(t, op.Resilience, "route %s %s should carry resilience", op.Method, op.Path) + require.NotNil(t, op.Resilience.Timeout) + assert.Equal(t, "45s", *op.Resilience.Timeout) + require.NotNil(t, op.Resilience.IdleTimeout) + assert.Equal(t, "0s", *op.Resilience.IdleTimeout) + } +} + +// With no resilience block on the source provider, no operation gets a resilience block (unchanged +// behavior — routes fall back to the gateway's global default timeout). +func TestTransformProvider_NoResilience_NotAttached(t *testing.T) { + transformer, _ := setupTestTransformer(t) + + provider := &api.LLMProviderConfiguration{ + ApiVersion: "gateway.api-platform.wso2.com/v1", + Kind: "LlmProvider", + Metadata: api.Metadata{Name: "openai-provider"}, + Spec: api.LLMProviderConfigData{ + DisplayName: "test", + Version: "v1.0", + Template: "openai", + Upstream: api.LLMProviderConfigData_Upstream{Url: stringPtr("https://api.example.com")}, + AccessControl: api.LLMAccessControl{Mode: api.AllowAll}, + }, + } + + result, err := transformer.Transform(provider, &api.RestAPI{}) + require.NoError(t, err) + require.NotEmpty(t, result.Spec.Operations) + + for _, op := range result.Spec.Operations { + assert.Nil(t, op.Resilience, "route %s %s should not carry resilience", op.Method, op.Path) + } +} + +// A proxy is always allow-all with no access control, so every generated route carries the +// resilience block. +func TestTransformProxy_ResilienceOnAllRoutes(t *testing.T) { + store := storage.NewConfigStore() + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + db := newTestSQLiteStorage(t, logger) + + template := &models.StoredLLMProviderTemplate{ + UUID: "0000-db-template-id-0000-000000000001", + Configuration: api.LLMProviderTemplate{ + ApiVersion: api.LLMProviderTemplateApiVersionGatewayApiPlatformWso2Comv1, + Kind: api.LLMProviderTemplateKindLlmProviderTemplate, + Metadata: api.Metadata{Name: "openai"}, + Spec: api.LLMProviderTemplateData{DisplayName: "openai"}, + }, + } + require.NoError(t, db.SaveLLMProviderTemplate(template)) + + now := time.Now() + provider := &models.StoredConfig{ + UUID: "0000-db-provider-id-0000-000000000000", + Kind: string(api.LLMProviderConfigurationKindLlmProvider), + Handle: "db-provider", + DisplayName: "db-provider", + Version: "v1.0", + Configuration: api.RestAPI{ + ApiVersion: api.RestAPIApiVersionGatewayApiPlatformWso2Comv1, + Kind: api.RestAPIKindRestApi, + Metadata: api.Metadata{Name: "db-provider"}, + Spec: api.APIConfigData{ + DisplayName: "db-provider", + Version: "v1.0", + Context: "/db-provider", + Upstream: struct { + Main api.Upstream `json:"main" yaml:"main"` + Sandbox *api.Upstream `json:"sandbox,omitempty" yaml:"sandbox,omitempty"` + }{Main: api.Upstream{Url: stringPtr("https://api.openai.com")}}, + }, + }, + SourceConfiguration: api.LLMProviderConfiguration{ + ApiVersion: api.LLMProviderConfigurationApiVersionGatewayApiPlatformWso2Comv1, + Kind: api.LLMProviderConfigurationKindLlmProvider, + Metadata: api.Metadata{Name: "db-provider"}, + Spec: api.LLMProviderConfigData{ + DisplayName: "db-provider", + Version: "v1.0", + Context: stringPtr("/db-provider"), + Template: "openai", + Upstream: api.LLMProviderConfigData_Upstream{Url: stringPtr("https://api.openai.com")}, + AccessControl: api.LLMAccessControl{Mode: api.AllowAll}, + }, + }, + DesiredState: models.StateDeployed, + CreatedAt: now, + UpdatedAt: now, + } + require.NoError(t, db.SaveConfig(provider)) + + routerConfig := &config.RouterConfig{ListenerPort: 8080} + transformer := NewLLMProviderTransformer(store, db, routerConfig, newTestPolicyVersionResolver()) + + timeout := "75s" + proxy := &api.LLMProxyConfiguration{ + ApiVersion: api.LLMProxyConfigurationApiVersionGatewayApiPlatformWso2Comv1, + Kind: api.LLMProxyConfigurationKindLlmProxy, + Metadata: api.Metadata{Name: "db-proxy"}, + Spec: api.LLMProxyConfigData{ + DisplayName: "db-proxy", + Version: "v1.0", + Provider: api.LLMProxyProvider{Id: "db-provider"}, + Resilience: &api.Resilience{Timeout: &timeout}, + }, + } + + result, err := transformer.Transform(proxy, &api.RestAPI{}) + require.NoError(t, err) + require.NotEmpty(t, result.Spec.Operations) + + for _, op := range result.Spec.Operations { + require.NotNil(t, op.Resilience, "proxy route %s %s should carry resilience", op.Method, op.Path) + require.NotNil(t, op.Resilience.Timeout) + assert.Equal(t, "75s", *op.Resilience.Timeout) + } +} diff --git a/gateway/gateway-controller/pkg/utils/llm_transformer.go b/gateway/gateway-controller/pkg/utils/llm_transformer.go index 0146d2ad64..d4e1409062 100644 --- a/gateway/gateway-controller/pkg/utils/llm_transformer.go +++ b/gateway/gateway-controller/pkg/utils/llm_transformer.go @@ -295,6 +295,9 @@ func (t *LLMProviderTransformer) transformProxy(proxy *api.LLMProxyConfiguration } } } + // A proxy is always allow-all with no access control, so there are no deny routes: + // attach API-level resilience to all generated routes. + applyResilienceToTrafficRoutes(ops, proxy.Spec.Resilience, nil) spec.Operations = ops // Global (api-level) policies: route into the derived RestAPI's spec.Policies so they are @@ -388,6 +391,12 @@ func (t *LLMProviderTransformer) transformProvider(provider *api.LLMProviderConf var ops []api.Operation + // denyOpKeys tracks the routes created purely to deny traffic (the access-control + // exception routes, which carry the 404 respond policy). API-level resilience is NOT + // attached to these (they never reach an upstream). Populated in allow_all mode; empty + // in deny_all (where every created route is an allowed/forwarding route). + denyOpKeys := make(map[pathMethodKey]bool) + switch mode { case api.AllowAll: var denyPolicyVersion string @@ -427,6 +436,7 @@ func (t *LLMProviderTransformer) transformProvider(provider *api.LLMProviderConf for _, method := range methods { key := pathMethodKey{path: ex.Path, method: method} deniedPathMethods[key] = true + denyOpKeys[key] = true // Check if operation exists if _, exists := operationRegistry[key]; !exists { @@ -634,6 +644,8 @@ func (t *LLMProviderTransformer) transformProvider(provider *api.LLMProviderConf } } } + // Attach API-level resilience to every traffic-forwarding route, skipping the deny routes. + applyResilienceToTrafficRoutes(ops, provider.Spec.Resilience, denyOpKeys) spec.Operations = ops // Global (api-level) policies: route into the derived RestAPI's spec.Policies so they are @@ -658,6 +670,27 @@ func (t *LLMProviderTransformer) transformProvider(provider *api.LLMProviderConf return output, nil } +// applyResilienceToTrafficRoutes attaches the API-level resilience block to every operation that +// forwards traffic upstream, skipping access-control deny routes (which only return a canned 404 +// and never reach an upstream, so a timeout on them is meaningless). denyKeys identifies the deny +// routes by path+method; it is empty for deny_all and proxy (no deny routes exist there), so in +// those cases the block is applied to all routes. A nil resilience block is a no-op. +// +// Resilience is attached at the operation level (not the derived API level) on purpose: that keeps +// the deny routes on the gateway's global default timeout instead of inheriting the API-level value +// via the translator's per-field fallback. +func applyResilienceToTrafficRoutes(ops []api.Operation, resilience *api.Resilience, denyKeys map[pathMethodKey]bool) { + if resilience == nil { + return + } + for i := range ops { + if denyKeys[pathMethodKey{path: ops[i].Path, method: string(ops[i].Method)}] { + continue + } + ops[i].Resilience = resilience + } +} + // GetUpstreamAuthApikeyPolicyParams renders the policy params with given header and value func GetUpstreamAuthApikeyPolicyParams(header, value string) (map[string]interface{}, error) { rendered := fmt.Sprintf(constants.UPSTREAM_AUTH_APIKEY_POLICY_PARAMS, header, value) diff --git a/gateway/gateway-controller/pkg/xds/resilience_duration_test.go b/gateway/gateway-controller/pkg/xds/resilience_duration_test.go new file mode 100644 index 0000000000..714c99d83b --- /dev/null +++ b/gateway/gateway-controller/pkg/xds/resilience_duration_test.go @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2026, WSO2 LLC. (https://www.wso2.com). + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package xds + +import "testing" + +// parseDurationAllowZero must accept exactly what the CRD admission controller accepts +// (constants.ResilienceDurationPattern): single-unit durations including "0s" to disable, while +// rejecting compound, negative, and unitless values. +func TestParseDurationAllowZero_MatchesCRDPattern(t *testing.T) { + ptr := func(s string) *string { return &s } + + t.Run("accepts single-unit and zero", func(t *testing.T) { + for _, in := range []string{"30s", "500ms", "1m", "2h", "1.5s", "0s", "0ms"} { + d, err := parseDurationAllowZero(ptr(in)) + if err != nil { + t.Errorf("expected %q to be accepted, got error: %v", in, err) + continue + } + if d == nil { + t.Errorf("expected %q to yield a non-nil duration", in) + } + } + }) + + t.Run("nil and empty yield nil without error", func(t *testing.T) { + for _, in := range []*string{nil, ptr(""), ptr(" ")} { + d, err := parseDurationAllowZero(in) + if err != nil || d != nil { + t.Errorf("expected nil,nil for empty input, got %v,%v", d, err) + } + } + }) + + t.Run("rejects compound, negative, and unitless", func(t *testing.T) { + for _, in := range []string{"1h30m", "1m30s", "-30s", "-5s", "30", "0", "15seconds", "abc"} { + if _, err := parseDurationAllowZero(ptr(in)); err == nil { + t.Errorf("expected %q to be rejected, but it was accepted", in) + } + } + }) +} diff --git a/gateway/gateway-controller/pkg/xds/translator.go b/gateway/gateway-controller/pkg/xds/translator.go index 408a6dda7a..6943324b64 100644 --- a/gateway/gateway-controller/pkg/xds/translator.go +++ b/gateway/gateway-controller/pkg/xds/translator.go @@ -96,9 +96,11 @@ type Translator struct { } // resolvedTimeout represents parsed timeout values for an upstream. -// Currently only connect timeout is supported at the upstream definition level. +// Route and Idle come from the resilience block. type resolvedTimeout struct { Connect *time.Duration + Route *time.Duration + Idle *time.Duration } // NewTranslator creates a new translator @@ -214,6 +216,16 @@ func (t *Translator) translateRuntimeConfig(rdc *models.RuntimeDeployConfig) ([] return routes, clusters, nil } +// routeTimeoutOrDefault returns the per-route timeout when configured (including an +// explicit zero, which disables the timeout in Envoy), otherwise the global default +// expressed in milliseconds. +func (t *Translator) routeTimeoutOrDefault(v *time.Duration, defaultMs uint32) *durationpb.Duration { + if v != nil { + return durationpb.New(*v) + } + return durationpb.New(time.Duration(defaultMs) * time.Millisecond) +} + // createRouteFromRDC creates an Envoy route from a RuntimeDeployConfig Route. func (t *Translator) createRouteFromRDC(routeKey string, rdcRoute *models.Route, rdc *models.RuntimeDeployConfig) *route.Route { fullPath := rdcRoute.Path @@ -249,15 +261,17 @@ func (t *Translator) createRouteFromRDC(routeKey string, rdcRoute *models.Route, } } - // Build route action with timeouts + // Build route action with timeouts. Per-route resilience values (from the API/operation + // resilience block) take precedence; otherwise fall back to the global route defaults. + var routeResilienceTimeout, routeResilienceIdle *time.Duration + if rdcRoute.Timeout != nil { + routeResilienceTimeout = rdcRoute.Timeout.Timeout + routeResilienceIdle = rdcRoute.Timeout.IdleTimeout + } routeAction := &route.Route_Route{ Route: &route.RouteAction{ - Timeout: durationpb.New( - time.Duration(t.routerConfig.Upstream.Timeouts.RouteTimeoutMs) * time.Millisecond, - ), - IdleTimeout: durationpb.New( - time.Duration(t.routerConfig.Upstream.Timeouts.RouteIdleTimeoutMs) * time.Millisecond, - ), + Timeout: t.routeTimeoutOrDefault(routeResilienceTimeout, t.routerConfig.Upstream.Timeouts.RouteTimeoutMs), + IdleTimeout: t.routeTimeoutOrDefault(routeResilienceIdle, t.routerConfig.Upstream.Timeouts.RouteIdleTimeoutMs), }, } @@ -818,13 +832,25 @@ func (t *Translator) translateAPIConfig(cfg *models.StoredConfig, allConfigs []* } } + // Resolve API-level resilience timeouts once; operation-level values override per field. + apiTimeout, apiIdleTimeout, err := ResolveResilience(apiData.Resilience) + if err != nil { + return nil, nil, fmt.Errorf("invalid API-level resilience: %w", err) + } + for _, op := range apiData.Operations { // Determine if dynamic cluster selection should be used // When upstreamDefinitions exist, use cluster_header routing so policies can select the upstream useClusterHeader := apiData.UpstreamDefinitions != nil && len(*apiData.UpstreamDefinitions) > 0 + opTimeout, opIdleTimeout, err := ResolveResilience(op.Resilience) + if err != nil { + return nil, nil, fmt.Errorf("invalid resilience for operation %s %s: %w", op.Method, op.Path, err) + } + opTimeoutCfg := combineRouteResilience(mainTimeout, apiTimeout, apiIdleTimeout, opTimeout, opIdleTimeout) + r := t.createRoute(cfg.UUID, apiData.DisplayName, apiData.Version, apiData.Context, string(op.Method), op.Path, - mainClusterName, parsedMainURL.Path, effectiveMainVHost, cfg.Kind, templateHandle, providerName, apiData.Upstream.Main.HostRewrite, apiProjectID, mainTimeout, useClusterHeader, upstreamDefPaths) + mainClusterName, parsedMainURL.Path, effectiveMainVHost, cfg.Kind, templateHandle, providerName, apiData.Upstream.Main.HostRewrite, apiProjectID, opTimeoutCfg, useClusterHeader, upstreamDefPaths) mainRoutesList = append(mainRoutesList, r) } routesList = append(routesList, mainRoutesList...) @@ -850,8 +876,14 @@ func (t *Translator) translateAPIConfig(cfg *models.StoredConfig, allConfigs []* sbRoutesList := make([]*route.Route, 0) sbUseClusterHeader := apiData.UpstreamDefinitions != nil && len(*apiData.UpstreamDefinitions) > 0 for _, op := range apiData.Operations { + opTimeout, opIdleTimeout, err := ResolveResilience(op.Resilience) + if err != nil { + return nil, nil, fmt.Errorf("invalid resilience for operation %s %s: %w", op.Method, op.Path, err) + } + opTimeoutCfg := combineRouteResilience(sbTimeout, apiTimeout, apiIdleTimeout, opTimeout, opIdleTimeout) + r := t.createRoute(cfg.UUID, apiData.DisplayName, apiData.Version, apiData.Context, string(op.Method), op.Path, - sbClusterName, parsedSbURL.Path, effectiveSandboxVHost, cfg.Kind, templateHandle, providerName, apiData.Upstream.Sandbox.HostRewrite, apiProjectID, sbTimeout, sbUseClusterHeader, upstreamDefPaths) + sbClusterName, parsedSbURL.Path, effectiveSandboxVHost, cfg.Kind, templateHandle, providerName, apiData.Upstream.Sandbox.HostRewrite, apiProjectID, opTimeoutCfg, sbUseClusterHeader, upstreamDefPaths) sbRoutesList = append(sbRoutesList, r) } routesList = append(routesList, sbRoutesList...) @@ -1042,6 +1074,13 @@ func (t *Translator) createListener(virtualHosts []*route.VirtualHost, isHTTPS b HttpFilters: httpFilters, ServerHeaderTransformation: convertServerHeaderTransformation(t.routerConfig.HTTPListener.ServerHeaderTransformation), ServerName: t.routerConfig.HTTPListener.ServerHeaderValue, + // HCM-level (downstream) timeouts. Defaults match Envoy's documented defaults. + RequestTimeout: durationpb.New(t.routerConfig.HTTPListener.Timeouts.RequestTimeout), + RequestHeadersTimeout: durationpb.New(t.routerConfig.HTTPListener.Timeouts.RequestHeadersTimeout), + StreamIdleTimeout: durationpb.New(t.routerConfig.HTTPListener.Timeouts.StreamIdleTimeout), + CommonHttpProtocolOptions: &core.HttpProtocolOptions{ + IdleTimeout: durationpb.New(t.routerConfig.HTTPListener.Timeouts.IdleTimeout), + }, } // Add access logs if enabled @@ -1791,15 +1830,17 @@ func (t *Translator) createRoute(apiId, apiName, apiVersion, context, method, pa } } - // Currently the route level timeouts are configurable using the global configuration. + // Route-level timeouts: per-route resilience values (resilience block) take precedence, + // otherwise fall back to the global configuration defaults. + var routeTimeout, routeIdleTimeout *time.Duration + if timeoutCfg != nil { + routeTimeout = timeoutCfg.Route + routeIdleTimeout = timeoutCfg.Idle + } routeAction := &route.Route_Route{ Route: &route.RouteAction{ - Timeout: durationpb.New( - time.Duration(t.routerConfig.Upstream.Timeouts.RouteTimeoutMs) * time.Millisecond, - ), - IdleTimeout: durationpb.New( - time.Duration(t.routerConfig.Upstream.Timeouts.RouteIdleTimeoutMs) * time.Millisecond, - ), + Timeout: t.routeTimeoutOrDefault(routeTimeout, t.routerConfig.Upstream.Timeouts.RouteTimeoutMs), + IdleTimeout: t.routeTimeoutOrDefault(routeIdleTimeout, t.routerConfig.Upstream.Timeouts.RouteIdleTimeoutMs), }, } @@ -3034,6 +3075,68 @@ func parseTimeout(timeoutStr *string) (*time.Duration, error) { return &duration, nil } +// parseDurationAllowZero parses a duration string (e.g. "15s", "0s") into a *time.Duration. +// Unlike parseTimeout it accepts zero ("0s" means the timeout is explicitly disabled). The format +// is enforced against constants.ResilienceDurationPattern so this downstream parser stays +// consistent with the CRD admission controller and the management-API validator: compound +// ("1h30m"), negative ("-30s"), and unitless ("0") values are rejected. Returns nil for nil/empty. +func parseDurationAllowZero(timeoutStr *string) (*time.Duration, error) { + if timeoutStr == nil || strings.TrimSpace(*timeoutStr) == "" { + return nil, nil + } + + s := strings.TrimSpace(*timeoutStr) + if !constants.ResilienceDurationRegex.MatchString(s) { + return nil, fmt.Errorf("invalid timeout format %q: expected a single-unit duration like \"30s\", \"500ms\", or \"0s\" to disable", s) + } + + duration, err := time.ParseDuration(s) + if err != nil { + return nil, fmt.Errorf("invalid timeout format: %w", err) + } + + return &duration, nil +} + +// ResolveResilience parses a resilience block into route timeout and idle-timeout durations. +// A nil block, or unset fields, yield nil durations (meaning "use the global default"). +// "0s" yields a non-nil zero duration (meaning "explicitly disabled"). +func ResolveResilience(r *api.Resilience) (timeout *time.Duration, idleTimeout *time.Duration, err error) { + if r == nil { + return nil, nil, nil + } + if timeout, err = parseDurationAllowZero(r.Timeout); err != nil { + return nil, nil, fmt.Errorf("invalid resilience.timeout: %w", err) + } + if idleTimeout, err = parseDurationAllowZero(r.IdleTimeout); err != nil { + return nil, nil, fmt.Errorf("invalid resilience.idleTimeout: %w", err) + } + return timeout, idleTimeout, nil +} + +// combineRouteResilience returns a resolvedTimeout for a single route, preserving the +// upstream connect timeout from base and applying the effective route/idle timeouts +// (operation-level overriding API-level, per field). It returns base unchanged when no +// resilience is configured at either level. +func combineRouteResilience(base *resolvedTimeout, apiTimeout, apiIdle, opTimeout, opIdle *time.Duration) *resolvedTimeout { + effTimeout := opTimeout + if effTimeout == nil { + effTimeout = apiTimeout + } + effIdle := opIdle + if effIdle == nil { + effIdle = apiIdle + } + if effTimeout == nil && effIdle == nil { + return base + } + rt := resolvedTimeout{Route: effTimeout, Idle: effIdle} + if base != nil { + rt.Connect = base.Connect + } + return &rt +} + // resolveTimeoutFromDefinition converts an UpstreamDefinition's timeout block into a resolvedTimeout. // Returns nil if there is no timeout block or all fields are empty. func resolveTimeoutFromDefinition(def *api.UpstreamDefinition) (*resolvedTimeout, error) { diff --git a/gateway/gateway-controller/pkg/xds/translator_test.go b/gateway/gateway-controller/pkg/xds/translator_test.go index dc637583e8..2a70fe5715 100644 --- a/gateway/gateway-controller/pkg/xds/translator_test.go +++ b/gateway/gateway-controller/pkg/xds/translator_test.go @@ -31,6 +31,7 @@ import ( core "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" listener "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" route "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + hcm "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" tlsv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" resource "github.com/envoyproxy/go-control-plane/pkg/resource/v3" "github.com/stretchr/testify/assert" @@ -738,6 +739,53 @@ func TestTranslator_WildcardUpstreamRewriteFromRDC(t *testing.T) { } } +// TestTranslator_RouteResilienceTimeoutsFromRDC verifies that per-route resilience +// timeouts on a models.Route flow into the Envoy RouteAction, with fallback to the +// global defaults (60s / 300s from testRouterConfig) when unset, and that an explicit +// 0s is preserved (disables the timeout). +func TestTranslator_RouteResilienceTimeoutsFromRDC(t *testing.T) { + logger := createTestLogger() + routerCfg := testRouterConfig() + cfg := testConfig() + translator := NewTranslator(logger, routerCfg, nil, cfg) + + dur := func(d time.Duration) *time.Duration { return &d } + + tests := []struct { + name string + timeout *models.RouteTimeout + wantTimeout time.Duration + wantIdle time.Duration + }{ + {name: "nil timeout uses global defaults", timeout: nil, wantTimeout: 60 * time.Second, wantIdle: 300 * time.Second}, + {name: "configured values applied", timeout: &models.RouteTimeout{Timeout: dur(2 * time.Second), IdleTimeout: dur(10 * time.Second)}, wantTimeout: 2 * time.Second, wantIdle: 10 * time.Second}, + {name: "timeout set, idle falls back", timeout: &models.RouteTimeout{Timeout: dur(3 * time.Second)}, wantTimeout: 3 * time.Second, wantIdle: 300 * time.Second}, + {name: "explicit 0s disables route timeout", timeout: &models.RouteTimeout{Timeout: dur(0)}, wantTimeout: 0, wantIdle: 300 * time.Second}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + rdc := &models.RuntimeDeployConfig{ + UpstreamClusters: map[string]*models.UpstreamCluster{ + "main": {Endpoints: []models.Endpoint{{Host: "echo", Port: 80}}}, + }, + } + rdcRoute := &models.Route{ + Method: "GET", + Path: "/api/v1.0/items", + OperationPath: "/items", + AutoHostRewrite: true, + Timeout: tt.timeout, + Upstream: models.RouteUpstream{ClusterKey: "main"}, + } + r := translator.createRouteFromRDC("GET|/api/v1.0/items|", rdcRoute, rdc) + require.NotNil(t, r) + assert.Equal(t, tt.wantTimeout, r.GetRoute().GetTimeout().AsDuration(), "route timeout") + assert.Equal(t, tt.wantIdle, r.GetRoute().GetIdleTimeout().AsDuration(), "route idle timeout") + }) + } +} + func TestTranslator_SanitizeClusterName(t *testing.T) { logger := createTestLogger() routerCfg := testRouterConfig() @@ -948,6 +996,55 @@ func TestTranslator_ExtractProviderName_NilSourceConfig(t *testing.T) { assert.Equal(t, "", result) } +// extractHCM pulls the HttpConnectionManager out of the listener's first filter chain. +func extractHCM(t *testing.T, lis *listener.Listener) *hcm.HttpConnectionManager { + t.Helper() + require.NotEmpty(t, lis.GetFilterChains()) + require.NotEmpty(t, lis.GetFilterChains()[0].GetFilters()) + typedConfig := lis.GetFilterChains()[0].GetFilters()[0].GetTypedConfig() + require.NotNil(t, typedConfig) + manager := &hcm.HttpConnectionManager{} + require.NoError(t, typedConfig.UnmarshalTo(manager)) + return manager +} + +func TestTranslator_CreateListener_HCMTimeouts(t *testing.T) { + tests := []struct { + name string + timeouts config.HCMTimeouts + }{ + { + name: "configured values", + timeouts: config.HCMTimeouts{RequestTimeout: 30 * time.Second, RequestHeadersTimeout: 10 * time.Second, StreamIdleTimeout: 2 * time.Minute, IdleTimeout: 30 * time.Minute}, + }, + { + name: "envoy defaults flow through unchanged", + timeouts: config.HCMTimeouts{RequestTimeout: 0, RequestHeadersTimeout: 0, StreamIdleTimeout: 5 * time.Minute, IdleTimeout: time.Hour}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + logger := createTestLogger() + routerCfg := testRouterConfig() + routerCfg.HTTPListener.Timeouts = tt.timeouts + cfg := testConfig() + cfg.Router = *routerCfg + translator := NewTranslator(logger, routerCfg, nil, cfg) + + lis, _, err := translator.createListener(nil, false) + require.NoError(t, err) + + manager := extractHCM(t, lis) + assert.Equal(t, tt.timeouts.RequestTimeout, manager.GetRequestTimeout().AsDuration(), "request_timeout") + assert.Equal(t, tt.timeouts.RequestHeadersTimeout, manager.GetRequestHeadersTimeout().AsDuration(), "request_headers_timeout") + assert.Equal(t, tt.timeouts.StreamIdleTimeout, manager.GetStreamIdleTimeout().AsDuration(), "stream_idle_timeout") + require.NotNil(t, manager.GetCommonHttpProtocolOptions(), "common_http_protocol_options must be set") + assert.Equal(t, tt.timeouts.IdleTimeout, manager.GetCommonHttpProtocolOptions().GetIdleTimeout().AsDuration(), "idle_timeout") + }) + } +} + func TestTranslator_CreateAccessLogConfig_Disabled(t *testing.T) { // Note: createAccessLogConfig should only be called when access logs are enabled. // The check for enabled is done at the caller level. When called directly with disabled diff --git a/gateway/it/features/backend-timeout.feature b/gateway/it/features/backend-timeout.feature new file mode 100644 index 0000000000..d701f21f4d --- /dev/null +++ b/gateway/it/features/backend-timeout.feature @@ -0,0 +1,135 @@ +# -------------------------------------------------------------------- +# Copyright (c) 2026, WSO2 LLC. (https://www.wso2.com). +# +# WSO2 LLC. licenses this file to you under the Apache License, +# Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# -------------------------------------------------------------------- + +@backend-timeout @resilience +Feature: Backend route timeouts via the resilience block + As an API developer + I want to configure the route timeout via a resilience block at the API and operation level + So that requests to slow backends are terminated by the gateway within the configured time + + # These scenarios exercise resilience.timeout (Envoy RouteAction.Timeout). The slow + # backend is httpbin (echo-backend) /delay/{n}, which sleeps n seconds before responding; + # when the route timeout is shorter than the delay, the gateway returns 504. + # resilience.idleTimeout maps to RouteAction.IdleTimeout and is covered by unit tests + # (it cannot be exercised deterministically over HTTP here). + + Background: + Given the gateway services are running + + # API-level resilience.timeout (2s) is shorter than the backend delay (5s), so the + # gateway must time the route out with 504 at ~2s instead of waiting for the backend. + Scenario: API-level resilience timeout terminates a slow backend + Given I authenticate using basic auth as "admin" + When I deploy this API configuration: + """ + apiVersion: gateway.api-platform.wso2.com/v1 + kind: RestApi + metadata: + name: backend-timeout-api-v1.0 + spec: + displayName: Backend-Timeout-API + version: v1.0 + context: /backend-timeout-api/$version + upstream: + main: + url: http://echo-backend:80 + resilience: + timeout: 2s + operations: + - method: GET + path: /get + - method: GET + path: /delay/5 + """ + Then the response should be successful + And I wait for the endpoint "http://localhost:8080/backend-timeout-api/v1.0/get" to be ready + And I record the current time as "request_start" + When I send a GET request to "http://localhost:8080/backend-timeout-api/v1.0/delay/5" + Then the response status code should be 504 + And the request should have taken at least "2" seconds since "request_start" + Given I authenticate using basic auth as "admin" + When I delete the API "backend-timeout-api-v1.0" + Then the response should be successful + + # Operation-level resilience overrides the API level: the API-level timeout (10s) is + # longer than the backend delay (5s) and would let the request succeed, but the + # operation-level timeout (2s) wins, so the gateway returns 504 at ~2s. + Scenario: Operation-level resilience timeout overrides the API level + Given I authenticate using basic auth as "admin" + When I deploy this API configuration: + """ + apiVersion: gateway.api-platform.wso2.com/v1 + kind: RestApi + metadata: + name: backend-timeout-override-v1.0 + spec: + displayName: Backend-Timeout-Override + version: v1.0 + context: /backend-timeout-override/$version + upstream: + main: + url: http://echo-backend:80 + resilience: + timeout: 10s + operations: + - method: GET + path: /get + - method: GET + path: /delay/5 + resilience: + timeout: 2s + """ + Then the response should be successful + And I wait for the endpoint "http://localhost:8080/backend-timeout-override/v1.0/get" to be ready + And I record the current time as "request_start" + When I send a GET request to "http://localhost:8080/backend-timeout-override/v1.0/delay/5" + Then the response status code should be 504 + And the request should have taken at least "2" seconds since "request_start" + Given I authenticate using basic auth as "admin" + When I delete the API "backend-timeout-override-v1.0" + Then the response should be successful + + # Without a resilience block the global route timeout default (60s) applies, so a + # backend that responds within it succeeds normally. + Scenario: No resilience block falls back to the global default and succeeds + Given I authenticate using basic auth as "admin" + When I deploy this API configuration: + """ + apiVersion: gateway.api-platform.wso2.com/v1 + kind: RestApi + metadata: + name: backend-timeout-default-v1.0 + spec: + displayName: Backend-Timeout-Default + version: v1.0 + context: /backend-timeout-default/$version + upstream: + main: + url: http://echo-backend:80 + operations: + - method: GET + path: /get + - method: GET + path: /delay/2 + """ + Then the response should be successful + And I wait for the endpoint "http://localhost:8080/backend-timeout-default/v1.0/get" to be ready + When I send a GET request to "http://localhost:8080/backend-timeout-default/v1.0/delay/2" + Then the response status code should be 200 + Given I authenticate using basic auth as "admin" + When I delete the API "backend-timeout-default-v1.0" + Then the response should be successful diff --git a/gateway/it/features/llm-backend-timeout.feature b/gateway/it/features/llm-backend-timeout.feature new file mode 100644 index 0000000000..addc30347b --- /dev/null +++ b/gateway/it/features/llm-backend-timeout.feature @@ -0,0 +1,228 @@ +# -------------------------------------------------------------------- +# Copyright (c) 2026, WSO2 LLC. (https://www.wso2.com). +# +# WSO2 LLC. licenses this file to you under the Apache License, +# Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# -------------------------------------------------------------------- + +@llm @backend-timeout @resilience +Feature: Backend route timeouts for LLM providers and proxies via the resilience block + As an API developer + I want to configure the route timeout via an API-level resilience block on LLM providers and proxies + So that requests to slow LLM backends are terminated by the gateway within the configured time + + # LLM kinds support resilience at the API level only + + Background: + Given the gateway services are running + + # allow_all creates a catch-all route for all traffic; the API-level resilience.timeout (2s) is + # shorter than the backend delay (5s), so the gateway times the route out with 504 at ~2s. + Scenario: LLM provider API-level resilience timeout terminates a slow backend + Given I authenticate using basic auth as "admin" + When I create this LLM provider: + """ + apiVersion: gateway.api-platform.wso2.com/v1 + kind: LlmProvider + metadata: + name: llm-timeout-provider + spec: + displayName: LLM Timeout Provider + version: v1.0 + template: openai + context: /llm-timeout + upstream: + url: http://echo-backend:80 + accessControl: + mode: allow_all + resilience: + timeout: 2s + """ + Then the response status code should be 201 + And I wait for the endpoint "http://localhost:8080/llm-timeout/get" to be ready + And I record the current time as "request_start" + When I send a GET request to "http://localhost:8080/llm-timeout/delay/5" + Then the response status code should be 504 + And the request should have taken at least "2" seconds since "request_start" + Given I authenticate using basic auth as "admin" + When I delete the LLM provider "llm-timeout-provider" + Then the response should be successful + + # deny_all creates routes only for the allow-listed exception paths; the resilience block must be + # attached to those (forwarding) routes too. The allow-listed /delay/5 route times out at ~2s. + Scenario: LLM provider deny_all attaches the timeout to allow-listed routes + Given I authenticate using basic auth as "admin" + When I create this LLM provider: + """ + apiVersion: gateway.api-platform.wso2.com/v1 + kind: LlmProvider + metadata: + name: llm-timeout-deny-provider + spec: + displayName: LLM Timeout Deny Provider + version: v1.0 + template: openai + context: /llm-timeout-deny + upstream: + url: http://echo-backend:80 + accessControl: + mode: deny_all + exceptions: + - path: /get + methods: [GET] + - path: /delay/5 + methods: [GET] + resilience: + timeout: 2s + """ + Then the response status code should be 201 + And I wait for the endpoint "http://localhost:8080/llm-timeout-deny/get" to be ready + And I record the current time as "request_start" + When I send a GET request to "http://localhost:8080/llm-timeout-deny/delay/5" + Then the response status code should be 504 + And the request should have taken at least "2" seconds since "request_start" + Given I authenticate using basic auth as "admin" + When I delete the LLM provider "llm-timeout-deny-provider" + Then the response should be successful + + # Without a resilience block the global route timeout default (60s) applies, so a backend that + # responds within it succeeds normally. + Scenario: LLM provider without a resilience block falls back to the global default and succeeds + Given I authenticate using basic auth as "admin" + When I create this LLM provider: + """ + apiVersion: gateway.api-platform.wso2.com/v1 + kind: LlmProvider + metadata: + name: llm-timeout-default-provider + spec: + displayName: LLM Timeout Default Provider + version: v1.0 + template: openai + context: /llm-timeout-default + upstream: + url: http://echo-backend:80 + accessControl: + mode: allow_all + """ + Then the response status code should be 201 + And I wait for the endpoint "http://localhost:8080/llm-timeout-default/get" to be ready + When I send a GET request to "http://localhost:8080/llm-timeout-default/delay/2" + Then the response status code should be 200 + Given I authenticate using basic auth as "admin" + When I delete the LLM provider "llm-timeout-default-provider" + Then the response should be successful + + # A proxy is always allow-all (catch-all). The proxy's own API-level resilience.timeout (2s) + # bounds the whole proxied call (client -> proxy route -> loopback -> provider route -> backend). + # The backing provider is left on the global default, so the 504 is attributable to the proxy. + Scenario: LLM proxy API-level resilience timeout terminates a slow backend + Given I authenticate using basic auth as "admin" + When I create this LLM provider: + """ + apiVersion: gateway.api-platform.wso2.com/v1 + kind: LlmProvider + metadata: + name: llm-timeout-proxy-provider + spec: + displayName: LLM Timeout Proxy Provider + version: v1.0 + template: openai + context: /llm-timeout-backing + upstream: + url: http://echo-backend:80 + accessControl: + mode: allow_all + """ + Then the response status code should be 201 + When I deploy this LLM proxy configuration: + """ + apiVersion: gateway.api-platform.wso2.com/v1 + kind: LlmProxy + metadata: + name: llm-timeout-proxy + spec: + displayName: LLM Timeout Proxy + version: v1.0 + context: /llm-timeout-proxy + provider: + id: llm-timeout-proxy-provider + resilience: + timeout: 2s + """ + Then the response status code should be 201 + And I wait for the endpoint "http://localhost:8080/llm-timeout-proxy/get" to be ready + And I record the current time as "request_start" + When I send a GET request to "http://localhost:8080/llm-timeout-proxy/delay/5" + Then the response status code should be 504 + And the request should have taken at least "2" seconds since "request_start" + Given I authenticate using basic auth as "admin" + When I send a DELETE request to the "gateway-controller" service at "/llm-proxies/llm-timeout-proxy" + Then the response should be successful + Given I authenticate using basic auth as "admin" + When I delete the LLM provider "llm-timeout-proxy-provider" + Then the response should be successful + + # Both the proxy and its backing provider set their own resilience.timeout. The request traverses + # both routes (proxy 6s -> loopback -> provider 2s -> backend /delay/10), so the shorter inner + # provider timeout (2s) must fire first and bound the whole call. The upper-bound assertion proves + # the 2s provider timeout won, not the 6s proxy timeout. + Scenario: LLM proxy and provider each set resilience and the shorter (provider) timeout wins + Given I authenticate using basic auth as "admin" + When I create this LLM provider: + """ + apiVersion: gateway.api-platform.wso2.com/v1 + kind: LlmProvider + metadata: + name: llm-timeout-both-provider + spec: + displayName: LLM Timeout Both Provider + version: v1.0 + template: openai + context: /llm-timeout-both-backing + upstream: + url: http://echo-backend:80 + accessControl: + mode: allow_all + resilience: + timeout: 2s + """ + Then the response status code should be 201 + When I deploy this LLM proxy configuration: + """ + apiVersion: gateway.api-platform.wso2.com/v1 + kind: LlmProxy + metadata: + name: llm-timeout-both-proxy + spec: + displayName: LLM Timeout Both Proxy + version: v1.0 + context: /llm-timeout-both + provider: + id: llm-timeout-both-provider + resilience: + timeout: 6s + """ + Then the response status code should be 201 + And I wait for the endpoint "http://localhost:8080/llm-timeout-both/get" to be ready + And I record the current time as "request_start" + When I send a GET request to "http://localhost:8080/llm-timeout-both/delay/10" + Then the response status code should be 504 + And the request should have taken at least "2" seconds since "request_start" + And the request should have taken at most "4" seconds since "request_start" + Given I authenticate using basic auth as "admin" + When I send a DELETE request to the "gateway-controller" service at "/llm-proxies/llm-timeout-both-proxy" + Then the response should be successful + Given I authenticate using basic auth as "admin" + When I delete the LLM provider "llm-timeout-both-provider" + Then the response should be successful diff --git a/gateway/it/features/backend_timeout.feature b/gateway/it/features/upstream-connect-timeout.feature similarity index 62% rename from gateway/it/features/backend_timeout.feature rename to gateway/it/features/upstream-connect-timeout.feature index 655afa375d..ab855c50be 100644 --- a/gateway/it/features/backend_timeout.feature +++ b/gateway/it/features/upstream-connect-timeout.feature @@ -15,11 +15,15 @@ # under the License. # -------------------------------------------------------------------- -@backend-timeout -Feature: Backend timeout +@backend-timeout @timeouts +Feature: Timeouts As an API developer - I want backend timeout (upstreamDefinitions) to be enforced by the gateway - So that requests to slow or unreachable backends fail within the configured timeout + I want upstream (connect) and HTTP Connection Manager timeouts to be enforced by the gateway + So that requests to slow or unreachable backends, and slow downstream clients, + fail within the configured timeout + + # request_timeout, stream_idle_timeout and idle_timeout are not exercised here: + # small values would affect the whole shared suite Background: Given the gateway services are running @@ -43,7 +47,7 @@ Feature: Backend timeout timeout: connect: 6000ms upstreams: - - url: http://192.0.2.1:80 + - url: http://192.0.2.1:8080 upstream: main: ref: my-timeout-upstream @@ -54,6 +58,7 @@ Feature: Backend timeout Then the response should be successful And the response should be valid JSON And the JSON response field "status" should be "success" + And I wait for policy snapshot sync And I record the current time as "request_start" When I send a GET request to "http://localhost:8080/timeout-api/v1.0/" Then the response status code should be 503 @@ -78,7 +83,7 @@ Feature: Backend timeout upstreamDefinitions: - name: my-timeout-upstream-global upstreams: - - url: http://192.0.2.1:80 + - url: http://192.0.2.1:8080 upstream: main: ref: my-timeout-upstream-global @@ -89,6 +94,7 @@ Feature: Backend timeout Then the response should be successful And the response should be valid JSON And the JSON response field "status" should be "success" + And I wait for policy snapshot sync And I record the current time as "request_start" When I send a GET request to "http://localhost:8080/timeout-global/v1.0/" Then the response status code should be 503 @@ -97,3 +103,40 @@ Feature: Backend timeout When I delete the API "timeout-api-global-v1.0" Then the response should be successful + # Tests HCM request_headers_timeout (set to "5s" in it/test-config.toml). + # A raw client sends a partial request and never terminates the headers; the gateway + # must close the stream with 408 once request_headers_timeout elapses. + Scenario: HCM request_headers_timeout terminates a slow-header request + Given I authenticate using basic auth as "admin" + When I deploy this API configuration: + """ + apiVersion: gateway.api-platform.wso2.com/v1 + kind: RestApi + metadata: + name: headers-timeout-api-v1.0 + spec: + displayName: Headers-Timeout-API + version: v1.0 + context: /headers-timeout/$version + upstreamDefinitions: + - name: headers-timeout-upstream + upstreams: + - url: http://sample-backend:9080 + upstream: + main: + ref: headers-timeout-upstream + operations: + - method: GET + path: / + """ + Then the response should be successful + And the response should be valid JSON + And the JSON response field "status" should be "success" + And I record the current time as "request_start" + When I open a raw connection to "localhost:8080" and send incomplete request headers for path "/headers-timeout/v1.0/" + Then the raw response status code should be "408" + And the request should have taken at least "4" seconds since "request_start" + Given I authenticate using basic auth as "admin" + When I delete the API "headers-timeout-api-v1.0" + Then the response should be successful + diff --git a/gateway/it/steps_backend_timeout.go b/gateway/it/steps_backend_timeout.go deleted file mode 100644 index c5d8778acd..0000000000 --- a/gateway/it/steps_backend_timeout.go +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright (c) 2025, WSO2 LLC. (https://www.wso2.com). - * - * WSO2 LLC. licenses this file to you under the Apache License, - * Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package it - -import ( - "fmt" - "strconv" - "time" - - "github.com/cucumber/godog" -) - -// elapsedTimeToleranceSeconds is the tolerance when asserting minimum elapsed time (e.g. clock skew, scheduling). -const elapsedTimeToleranceSeconds = 1 - -// RegisterBackendTimeoutSteps registers step definitions for backend timeout scenarios -func RegisterBackendTimeoutSteps(ctx *godog.ScenarioContext, state *TestState) { - ctx.Step(`^I record the current time as "([^"]*)"$`, func(key string) error { - state.SetContextValue(key, time.Now()) - return nil - }) - - ctx.Step(`^the request should have taken at least "(\d+)" seconds since "([^"]*)"$`, func(expectedSecondsStr, key string) error { - expectedSeconds, err := strconv.Atoi(expectedSecondsStr) - if err != nil { - return fmt.Errorf("expected seconds must be a number, got: %s", expectedSecondsStr) - } - val, ok := state.GetContextValue(key) - if !ok { - return fmt.Errorf("no start time recorded; record the current time as \"request_start\" before sending the request") - } - start, ok := val.(time.Time) - if !ok { - return fmt.Errorf("no start time recorded; record the current time as %q before sending the request", key) - } - elapsed := time.Since(start) - minElapsed := time.Duration(expectedSeconds-elapsedTimeToleranceSeconds) * time.Second - if elapsed < minElapsed { - return fmt.Errorf("request should have taken at least %d seconds (with %ds tolerance), but elapsed time was %s", - expectedSeconds, elapsedTimeToleranceSeconds, elapsed.Round(time.Millisecond)) - } - return nil - }) -} diff --git a/gateway/it/steps_timeouts.go b/gateway/it/steps_timeouts.go new file mode 100644 index 0000000000..f8bea39830 --- /dev/null +++ b/gateway/it/steps_timeouts.go @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2025, WSO2 LLC. (https://www.wso2.com). + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package it + +import ( + "fmt" + "io" + "net" + "strconv" + "strings" + "time" + + "github.com/cucumber/godog" +) + +// elapsedTimeToleranceSeconds is the tolerance when asserting minimum elapsed time (e.g. clock skew, scheduling). +const elapsedTimeToleranceSeconds = 1 + +// rawResponseContextKey is where the slow-header raw response is stashed for later assertion. +const rawResponseContextKey = "raw_timeout_response" + +// RegisterTimeoutSteps registers step definitions for upstream and HCM timeout scenarios +func RegisterTimeoutSteps(ctx *godog.ScenarioContext, state *TestState) { + ctx.Step(`^I record the current time as "([^"]*)"$`, func(key string) error { + state.SetContextValue(key, time.Now()) + return nil + }) + + ctx.Step(`^the request should have taken at least "(\d+)" seconds since "([^"]*)"$`, func(expectedSecondsStr, key string) error { + expectedSeconds, err := strconv.Atoi(expectedSecondsStr) + if err != nil { + return fmt.Errorf("expected seconds must be a number, got: %s", expectedSecondsStr) + } + val, ok := state.GetContextValue(key) + if !ok { + return fmt.Errorf("no start time recorded; record the current time as \"request_start\" before sending the request") + } + start, ok := val.(time.Time) + if !ok { + return fmt.Errorf("no start time recorded; record the current time as %q before sending the request", key) + } + elapsed := time.Since(start) + minElapsed := time.Duration(expectedSeconds-elapsedTimeToleranceSeconds) * time.Second + if elapsed < minElapsed { + return fmt.Errorf("request should have taken at least %d seconds (with %ds tolerance), but elapsed time was %s", + expectedSeconds, elapsedTimeToleranceSeconds, elapsed.Round(time.Millisecond)) + } + return nil + }) + + ctx.Step(`^the request should have taken at most "(\d+)" seconds since "([^"]*)"$`, func(expectedSecondsStr, key string) error { + expectedSeconds, err := strconv.Atoi(expectedSecondsStr) + if err != nil { + return fmt.Errorf("expected seconds must be a number, got: %s", expectedSecondsStr) + } + val, ok := state.GetContextValue(key) + if !ok { + return fmt.Errorf("no start time recorded; record the current time as \"request_start\" before sending the request") + } + start, ok := val.(time.Time) + if !ok { + return fmt.Errorf("no start time recorded; record the current time as %q before sending the request", key) + } + elapsed := time.Since(start) + maxElapsed := time.Duration(expectedSeconds+elapsedTimeToleranceSeconds) * time.Second + if elapsed > maxElapsed { + return fmt.Errorf("request should have taken at most %d seconds (with %ds tolerance), but elapsed time was %s", + expectedSeconds, elapsedTimeToleranceSeconds, elapsed.Round(time.Millisecond)) + } + return nil + }) + + // Opens a raw TCP connection and sends a request line plus one header, but never + // terminates the header block (no final blank line). This forces the HCM + // request_headers_timeout to fire. The connection blocks until the gateway responds + // or closes it, and the raw response is stored for assertion. + ctx.Step(`^I open a raw connection to "([^"]*)" and send incomplete request headers for path "([^"]*)"$`, func(address, path string) error { + conn, err := net.DialTimeout("tcp", address, 10*time.Second) + if err != nil { + return fmt.Errorf("failed to connect to %s: %w", address, err) + } + defer conn.Close() + + // Request line + Host header, with NO terminating CRLF that ends the headers. + partial := fmt.Sprintf("GET %s HTTP/1.1\r\nHost: %s\r\n", path, address) + if _, err := conn.Write([]byte(partial)); err != nil { + return fmt.Errorf("failed to send partial request: %w", err) + } + + // Read until the gateway responds and closes, bounded by a deadline that is + // comfortably longer than request_headers_timeout so we capture the 408. + if err := conn.SetReadDeadline(time.Now().Add(30 * time.Second)); err != nil { + return fmt.Errorf("failed to set read deadline: %w", err) + } + raw, err := io.ReadAll(conn) + if err != nil && len(raw) == 0 { + return fmt.Errorf("failed to read response from gateway: %w", err) + } + state.SetContextValue(rawResponseContextKey, string(raw)) + return nil + }) + + ctx.Step(`^the raw response status code should be "([^"]*)"$`, func(expectedCode string) error { + val, ok := state.GetContextValue(rawResponseContextKey) + if !ok { + return fmt.Errorf("no raw response recorded; open a raw connection first") + } + raw, ok := val.(string) + if !ok { + return fmt.Errorf("recorded raw response has unexpected type %T", val) + } + statusLine := raw + if idx := strings.Index(raw, "\r\n"); idx >= 0 { + statusLine = raw[:idx] + } + if !strings.Contains(statusLine, expectedCode) { + return fmt.Errorf("expected raw response status line to contain %q, got %q", expectedCode, statusLine) + } + return nil + }) +} diff --git a/gateway/it/suite_test.go b/gateway/it/suite_test.go index 4cbcf0338e..d994bd8e6e 100644 --- a/gateway/it/suite_test.go +++ b/gateway/it/suite_test.go @@ -143,6 +143,9 @@ func getFeaturePaths() []string { "features/route-path-matching.feature", "features/secrets.feature", "features/template-functions.feature", + "features/upstream-connect-timeout.feature", + "features/backend-timeout.feature", + "features/llm-backend-timeout.feature", // Runs late: it restarts the gateway-controller (reject/reconnect scenario), so keep it // after features that assume an uninterrupted controller. Verifies the DP->CP artifact push. "features/dp-to-cp.feature", @@ -339,7 +342,7 @@ func InitializeScenario(ctx *godog.ScenarioContext) { RegisterMetricsSteps(ctx, testState, httpSteps) RegisterAuthSteps(ctx, testState, httpSteps) RegisterAPISteps(ctx, testState, httpSteps) - RegisterBackendTimeoutSteps(ctx, testState) + RegisterTimeoutSteps(ctx, testState) RegisterMCPSteps(ctx, testState, httpSteps, jwtSteps) RegisterLLMSteps(ctx, testState, httpSteps) RegisterJWTSteps(ctx, testState, httpSteps, jwtSteps) diff --git a/gateway/it/test-config.toml b/gateway/it/test-config.toml index 187d32390f..4833aee646 100644 --- a/gateway/it/test-config.toml +++ b/gateway/it/test-config.toml @@ -72,6 +72,10 @@ enabled = true enabled = true format = "text" +[router.http_listener.timeouts] +# This is the only HCM timeout set to a small, testable value here. +request_headers_timeout = "5s" + # ============================================================================= # POLICY ENGINE CONFIGURATION # ============================================================================= diff --git a/kubernetes/gateway-operator/api/v1alpha1/llmprovider_types.go b/kubernetes/gateway-operator/api/v1alpha1/llmprovider_types.go index 5d2fae5583..9535b86216 100644 --- a/kubernetes/gateway-operator/api/v1alpha1/llmprovider_types.go +++ b/kubernetes/gateway-operator/api/v1alpha1/llmprovider_types.go @@ -128,6 +128,11 @@ type LLMProviderConfigData struct { // Policies is the list of policies applied to this LLM provider. // +optional Policies []LLMPolicy `json:"policies,omitempty"` + + // Resilience configures API-level backend/route timeouts applied to all routes + // generated for this LLM provider. Supported at the API level only. + // +optional + Resilience *Resilience `json:"resilience,omitempty"` } // LLMPolicyPath defines a path/methods combination together with policy diff --git a/kubernetes/gateway-operator/api/v1alpha1/llmproxy_types.go b/kubernetes/gateway-operator/api/v1alpha1/llmproxy_types.go index 046f492af5..62aac873c9 100644 --- a/kubernetes/gateway-operator/api/v1alpha1/llmproxy_types.go +++ b/kubernetes/gateway-operator/api/v1alpha1/llmproxy_types.go @@ -64,6 +64,11 @@ type LLMProxyConfigData struct { // Policies is the list of policies applied to this LLM proxy. // +optional Policies []LLMPolicy `json:"policies,omitempty"` + + // Resilience configures API-level backend/route timeouts applied to all routes + // generated for this LLM proxy. Supported at the API level only. + // +optional + Resilience *Resilience `json:"resilience,omitempty"` } //+kubebuilder:object:root=true diff --git a/kubernetes/gateway-operator/api/v1alpha1/restapi_types.go b/kubernetes/gateway-operator/api/v1alpha1/restapi_types.go index 6077eb981f..a24bf15b78 100644 --- a/kubernetes/gateway-operator/api/v1alpha1/restapi_types.go +++ b/kubernetes/gateway-operator/api/v1alpha1/restapi_types.go @@ -74,6 +74,11 @@ type APIConfigData struct { // +optional Policies []Policy `json:"policies,omitempty"` + // Resilience API-level backend/route timeout configuration applied to all operations + // unless overridden at the operation level + // +optional + Resilience *Resilience `json:"resilience,omitempty"` + // Upstream API-level upstream configuration // +kubebuilder:validation:Required Upstream UpstreamConfig `json:"upstream"` @@ -125,6 +130,26 @@ type Operation struct { // Policies List of policies applied only to this operation (overrides or adds to API-level policies) // +optional Policies []Policy `json:"policies,omitempty"` + + // Resilience Operation-level backend/route timeout configuration (overrides API-level) + // +optional + Resilience *Resilience `json:"resilience,omitempty"` +} + +// Resilience defines backend/route timeout configuration (maps to Envoy RouteAction +// timeouts). Settable at the API level (applies to all routes) and/or the operation level +// (overrides the API level). "0s" disables a timeout; unset falls back to the gateway's +// global route timeout defaults. +type Resilience struct { + // Timeout Maximum time for the entire route (request to upstream response). "0s" disables. + // +optional + // +kubebuilder:validation:Pattern=`^\d+(\.\d+)?(ms|s|m|h)$` + Timeout *string `json:"timeout,omitempty"` + + // IdleTimeout Per-route stream idle timeout. "0s" disables. + // +optional + // +kubebuilder:validation:Pattern=`^\d+(\.\d+)?(ms|s|m|h)$` + IdleTimeout *string `json:"idleTimeout,omitempty"` } // OperationMethod HTTP method diff --git a/kubernetes/gateway-operator/api/v1alpha1/zz_generated.deepcopy.go b/kubernetes/gateway-operator/api/v1alpha1/zz_generated.deepcopy.go index b6163d5fd5..69e6912d3f 100644 --- a/kubernetes/gateway-operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/kubernetes/gateway-operator/api/v1alpha1/zz_generated.deepcopy.go @@ -43,6 +43,11 @@ func (in *APIConfigData) DeepCopyInto(out *APIConfigData) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.Resilience != nil { + in, out := &in.Resilience, &out.Resilience + *out = new(Resilience) + (*in).DeepCopyInto(*out) + } in.Upstream.DeepCopyInto(&out.Upstream) if in.Vhosts != nil { in, out := &in.Vhosts, &out.Vhosts @@ -797,6 +802,11 @@ func (in *LLMProviderConfigData) DeepCopyInto(out *LLMProviderConfigData) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.Resilience != nil { + in, out := &in.Resilience, &out.Resilience + *out = new(Resilience) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LLMProviderConfigData. @@ -987,6 +997,11 @@ func (in *LLMProxyConfigData) DeepCopyInto(out *LLMProxyConfigData) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.Resilience != nil { + in, out := &in.Resilience, &out.Resilience + *out = new(Resilience) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LLMProxyConfigData. @@ -1587,6 +1602,11 @@ func (in *Operation) DeepCopyInto(out *Operation) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.Resilience != nil { + in, out := &in.Resilience, &out.Resilience + *out = new(Resilience) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Operation. @@ -1624,6 +1644,31 @@ func (in *Policy) DeepCopy() *Policy { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Resilience) DeepCopyInto(out *Resilience) { + *out = *in + if in.Timeout != nil { + in, out := &in.Timeout, &out.Timeout + *out = new(string) + **out = **in + } + if in.IdleTimeout != nil { + in, out := &in.IdleTimeout, &out.IdleTimeout + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Resilience. +func (in *Resilience) DeepCopy() *Resilience { + if in == nil { + return nil + } + out := new(Resilience) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ResourceStatus) DeepCopyInto(out *ResourceStatus) { *out = *in diff --git a/kubernetes/gateway-operator/config/crd/bases/gateway.api-platform.wso2.com_llmproviders.yaml b/kubernetes/gateway-operator/config/crd/bases/gateway.api-platform.wso2.com_llmproviders.yaml index 8d48f6cbe8..78f1879087 100644 --- a/kubernetes/gateway-operator/config/crd/bases/gateway.api-platform.wso2.com_llmproviders.yaml +++ b/kubernetes/gateway-operator/config/crd/bases/gateway.api-platform.wso2.com_llmproviders.yaml @@ -153,6 +153,21 @@ spec: - version type: object type: array + resilience: + description: |- + Resilience configures API-level backend/route timeouts applied to all routes + generated for this LLM provider. Supported at the API level only. + properties: + idleTimeout: + description: IdleTimeout Per-route stream idle timeout. "0s" disables. + pattern: ^\d+(\.\d+)?(ms|s|m|h)$ + type: string + timeout: + description: Timeout Maximum time for the entire route (request + to upstream response). "0s" disables. + pattern: ^\d+(\.\d+)?(ms|s|m|h)$ + type: string + type: object template: description: Template is the LlmProviderTemplate name to apply. type: string diff --git a/kubernetes/gateway-operator/config/crd/bases/gateway.api-platform.wso2.com_llmproxies.yaml b/kubernetes/gateway-operator/config/crd/bases/gateway.api-platform.wso2.com_llmproxies.yaml index dd5a2a5579..81da02ebd1 100644 --- a/kubernetes/gateway-operator/config/crd/bases/gateway.api-platform.wso2.com_llmproxies.yaml +++ b/kubernetes/gateway-operator/config/crd/bases/gateway.api-platform.wso2.com_llmproxies.yaml @@ -180,6 +180,21 @@ spec: required: - id type: object + resilience: + description: |- + Resilience configures API-level backend/route timeouts applied to all routes + generated for this LLM proxy. Supported at the API level only. + properties: + idleTimeout: + description: IdleTimeout Per-route stream idle timeout. "0s" disables. + pattern: ^\d+(\.\d+)?(ms|s|m|h)$ + type: string + timeout: + description: Timeout Maximum time for the entire route (request + to upstream response). "0s" disables. + pattern: ^\d+(\.\d+)?(ms|s|m|h)$ + type: string + type: object version: description: Version is the semantic version of the LLM proxy. pattern: ^v?([0-9]+)(\.[0-9]+)?(\.[0-9]+)?(-[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?(\+[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$ diff --git a/kubernetes/gateway-operator/config/crd/bases/gateway.api-platform.wso2.com_restapis.yaml b/kubernetes/gateway-operator/config/crd/bases/gateway.api-platform.wso2.com_restapis.yaml index 3d1ad1a2aa..4e62b044a0 100644 --- a/kubernetes/gateway-operator/config/crd/bases/gateway.api-platform.wso2.com_restapis.yaml +++ b/kubernetes/gateway-operator/config/crd/bases/gateway.api-platform.wso2.com_restapis.yaml @@ -98,6 +98,21 @@ spec: - version type: object type: array + resilience: + description: Resilience Operation-level backend/route timeout + configuration (overrides API-level) + properties: + idleTimeout: + description: IdleTimeout Per-route stream idle timeout. + "0s" disables. + pattern: ^\d+(\.\d+)?(ms|s|m|h)$ + type: string + timeout: + description: Timeout Maximum time for the entire route (request + to upstream response). "0s" disables. + pattern: ^\d+(\.\d+)?(ms|s|m|h)$ + type: string + type: object required: - method - path @@ -132,6 +147,21 @@ spec: - version type: object type: array + resilience: + description: |- + Resilience API-level backend/route timeout configuration applied to all operations + unless overridden at the operation level + properties: + idleTimeout: + description: IdleTimeout Per-route stream idle timeout. "0s" disables. + pattern: ^\d+(\.\d+)?(ms|s|m|h)$ + type: string + timeout: + description: Timeout Maximum time for the entire route (request + to upstream response). "0s" disables. + pattern: ^\d+(\.\d+)?(ms|s|m|h)$ + type: string + type: object upstream: description: Upstream API-level upstream configuration properties: diff --git a/kubernetes/gateway-operator/config/samples/api_v1_restapi.yaml b/kubernetes/gateway-operator/config/samples/api_v1_restapi.yaml index 750e88cf08..74ce33658d 100644 --- a/kubernetes/gateway-operator/config/samples/api_v1_restapi.yaml +++ b/kubernetes/gateway-operator/config/samples/api_v1_restapi.yaml @@ -21,11 +21,18 @@ spec: # params: # issuers: # - WSO2KeyManager1 + # API-level resilience applies to all operations unless overridden per operation. + resilience: + timeout: 15s + idleTimeout: 0s operations: - method: GET path: /info - method: POST path: /submit + # Operation-level resilience overrides the API-level values for this route. + resilience: + timeout: 5s - method: GET path: /helloworld - method: GET diff --git a/kubernetes/helm/operator-helm-chart/crds/gateway.api-platform.wso2.com_llmproviders.yaml b/kubernetes/helm/operator-helm-chart/crds/gateway.api-platform.wso2.com_llmproviders.yaml index 8d48f6cbe8..78f1879087 100644 --- a/kubernetes/helm/operator-helm-chart/crds/gateway.api-platform.wso2.com_llmproviders.yaml +++ b/kubernetes/helm/operator-helm-chart/crds/gateway.api-platform.wso2.com_llmproviders.yaml @@ -153,6 +153,21 @@ spec: - version type: object type: array + resilience: + description: |- + Resilience configures API-level backend/route timeouts applied to all routes + generated for this LLM provider. Supported at the API level only. + properties: + idleTimeout: + description: IdleTimeout Per-route stream idle timeout. "0s" disables. + pattern: ^\d+(\.\d+)?(ms|s|m|h)$ + type: string + timeout: + description: Timeout Maximum time for the entire route (request + to upstream response). "0s" disables. + pattern: ^\d+(\.\d+)?(ms|s|m|h)$ + type: string + type: object template: description: Template is the LlmProviderTemplate name to apply. type: string diff --git a/kubernetes/helm/operator-helm-chart/crds/gateway.api-platform.wso2.com_llmproxies.yaml b/kubernetes/helm/operator-helm-chart/crds/gateway.api-platform.wso2.com_llmproxies.yaml index dd5a2a5579..81da02ebd1 100644 --- a/kubernetes/helm/operator-helm-chart/crds/gateway.api-platform.wso2.com_llmproxies.yaml +++ b/kubernetes/helm/operator-helm-chart/crds/gateway.api-platform.wso2.com_llmproxies.yaml @@ -180,6 +180,21 @@ spec: required: - id type: object + resilience: + description: |- + Resilience configures API-level backend/route timeouts applied to all routes + generated for this LLM proxy. Supported at the API level only. + properties: + idleTimeout: + description: IdleTimeout Per-route stream idle timeout. "0s" disables. + pattern: ^\d+(\.\d+)?(ms|s|m|h)$ + type: string + timeout: + description: Timeout Maximum time for the entire route (request + to upstream response). "0s" disables. + pattern: ^\d+(\.\d+)?(ms|s|m|h)$ + type: string + type: object version: description: Version is the semantic version of the LLM proxy. pattern: ^v?([0-9]+)(\.[0-9]+)?(\.[0-9]+)?(-[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?(\+[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$ diff --git a/kubernetes/helm/operator-helm-chart/crds/gateway.api-platform.wso2.com_restapis.yaml b/kubernetes/helm/operator-helm-chart/crds/gateway.api-platform.wso2.com_restapis.yaml index 3d1ad1a2aa..4e62b044a0 100644 --- a/kubernetes/helm/operator-helm-chart/crds/gateway.api-platform.wso2.com_restapis.yaml +++ b/kubernetes/helm/operator-helm-chart/crds/gateway.api-platform.wso2.com_restapis.yaml @@ -98,6 +98,21 @@ spec: - version type: object type: array + resilience: + description: Resilience Operation-level backend/route timeout + configuration (overrides API-level) + properties: + idleTimeout: + description: IdleTimeout Per-route stream idle timeout. + "0s" disables. + pattern: ^\d+(\.\d+)?(ms|s|m|h)$ + type: string + timeout: + description: Timeout Maximum time for the entire route (request + to upstream response). "0s" disables. + pattern: ^\d+(\.\d+)?(ms|s|m|h)$ + type: string + type: object required: - method - path @@ -132,6 +147,21 @@ spec: - version type: object type: array + resilience: + description: |- + Resilience API-level backend/route timeout configuration applied to all operations + unless overridden at the operation level + properties: + idleTimeout: + description: IdleTimeout Per-route stream idle timeout. "0s" disables. + pattern: ^\d+(\.\d+)?(ms|s|m|h)$ + type: string + timeout: + description: Timeout Maximum time for the entire route (request + to upstream response). "0s" disables. + pattern: ^\d+(\.\d+)?(ms|s|m|h)$ + type: string + type: object upstream: description: Upstream API-level upstream configuration properties: