diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 41262afd59..5ad870c6f0 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -997,7 +997,6 @@ jobs: caching-service: image: ghcr.io/balhar-jakub/caching-service:${{ github.run_id }}-${{ github.run_number }} env: - ZWE_CACHING_SERVICE_PERSISTENT: 'infinispan' CACHING_STORAGE_MODE: "infinispan" JGROUPS_BIND_ADDRESS: "caching-service" JGROUPS_BIND_PORT: "7600" @@ -2164,7 +2163,6 @@ jobs: caching-service: image: ghcr.io/balhar-jakub/caching-service:${{ github.run_id }}-${{ github.run_number }} env: - ZWE_CACHING_SERVICE_PERSISTENT: 'infinispan' JGROUPS_BIND_PORT: "7600" SERVER_SSL_KEYSTORETYPE: "PKCS12" CACHING_STORAGE_INFINISPAN_PERSISTENCE_DATALOCATION: "data_replica" @@ -2175,7 +2173,6 @@ jobs: caching-service-2: image: ghcr.io/balhar-jakub/caching-service:${{ github.run_id }}-${{ github.run_number }} env: - ZWE_CACHING_SERVICE_PERSISTENT: 'infinispan' JGROUPS_BIND_PORT: "7600" SERVER_SSL_KEYSTORETYPE: "PKCS12" CACHING_STORAGE_INFINISPAN_PERSISTENCE_DATALOCATION: "data" @@ -2250,13 +2247,12 @@ jobs: APIML_SECURITY_X509_ENABLED: true APIML_SECURITY_SSL_NONSTRICTVERIFYSSLCERTIFICATESOFSERVICES: true APIML_DISCOVERY_ALLPEERSURLS: https://apiml-2:10011/eureka,https://apiml:10011/eureka - ZWE_CACHING_SERVICE_PERSISTENT: 'infinispan' - JGROUPS_BIND_PORT: "7600" - SERVER_SSL_KEYSTORETYPE: "PKCS12" - CACHING_STORAGE_INFINISPAN_INITIALHOSTS: "apiml[7600],apiml-2[7600]" + CACHING_STORAGE_INFINISPAN_NUMSEGMENTS: "16" CACHING_STORAGE_MODE: "infinispan" + CACHING_STORAGE_INFINISPAN_INITIALHOSTS: "apiml[7600],apiml-2[7600]" + JGROUPS_BIND_PORT: "7600" JGROUPS_BIND_ADDRESS: "apiml" - CACHING_STORAGE_INFINISPAN_NUMSEGMENTS: "16" + JGROUPS_KEYEXCHANGE_PORT: 7601 apiml-2: image: ghcr.io/balhar-jakub/apiml:${{ github.run_id }}-${{ github.run_number }} env: @@ -2264,15 +2260,14 @@ jobs: APIML_GATEWAY_SERVICESTOLIMITREQUESTRATE: discoverableclient APIML_SECURITY_SSL_NONSTRICTVERIFYSSLCERTIFICATESOFSERVICES: true APIML_DISCOVERY_ALLPEERSURLS: https://apiml:10011/eureka,https://apiml-2:10011/eureka - ZWE_CACHING_SERVICE_PERSISTENT: 'infinispan' - JGROUPS_BIND_PORT: "7600" - SERVER_SSL_KEYSTORETYPE: "PKCS12" - CACHING_STORAGE_INFINISPAN_INITIALHOSTS: "apiml[7600],apiml-2[7600]" - CACHING_STORAGE_MODE: "infinispan" - JGROUPS_BIND_ADDRESS: "apiml-2" APIML_SERVICE_HOSTNAME: "apiml-2" logbackService: ZWEAGW2 CACHING_STORAGE_INFINISPAN_NUMSEGMENTS: "16" + CACHING_STORAGE_MODE: "infinispan" + CACHING_STORAGE_INFINISPAN_INITIALHOSTS: "apiml[7600],apiml-2[7600]" + JGROUPS_BIND_PORT: "7600" + JGROUPS_BIND_ADDRESS: "apiml-2" + JGROUPS_KEYEXCHANGE_PORT: 7601 api-catalog-services: image: ghcr.io/balhar-jakub/api-catalog-services:${{ github.run_id }}-${{ github.run_number }} volumes: diff --git a/apiml/src/main/java/org/zowe/apiml/ApimlApplication.java b/apiml/src/main/java/org/zowe/apiml/ApimlApplication.java index 174d94a367..cc9212973b 100644 --- a/apiml/src/main/java/org/zowe/apiml/ApimlApplication.java +++ b/apiml/src/main/java/org/zowe/apiml/ApimlApplication.java @@ -12,6 +12,7 @@ import org.springframework.boot.SpringApplication; import org.springframework.boot.actuate.autoconfigure.logging.OpenTelemetryLoggingAutoConfiguration; +import org.springframework.boot.actuate.autoconfigure.metrics.cache.CacheMetricsAutoConfiguration; import org.springframework.boot.actuate.autoconfigure.opentelemetry.OpenTelemetryAutoConfiguration; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.autoconfigure.security.oauth2.client.reactive.ReactiveOAuth2ClientAutoConfiguration; @@ -28,7 +29,8 @@ ReactiveOAuth2ClientAutoConfiguration.class, OpenTelemetryAutoConfiguration.class, OpenTelemetryLoggingAutoConfiguration.class, - io.opentelemetry.instrumentation.spring.autoconfigure.OpenTelemetryAutoConfiguration.class + io.opentelemetry.instrumentation.spring.autoconfigure.OpenTelemetryAutoConfiguration.class, + CacheMetricsAutoConfiguration.class }, scanBasePackages = { "org.zowe.apiml.filter", diff --git a/apiml/src/main/resources/application.yml b/apiml/src/main/resources/application.yml index 89e89b4496..9d04d8d6be 100644 --- a/apiml/src/main/resources/application.yml +++ b/apiml/src/main/resources/application.yml @@ -346,7 +346,9 @@ logging: org.apache.tomcat.util.net: DEBUG org.apache.tomcat.util.net.jsse.JSSESupport: INFO org.ehcache: INFO - org.jgroups: DEBUG + org.jgroups: TRACE + org.jgroups.protocols: ERROR + org.infinispan: DEBUG org.springframework.context.support.[PostProcessorRegistrationDelegate$BeanPostProcessorChecker]: DEBUG org.springframework.security: TRACE org.springframework.security.web: TRACE diff --git a/caching-service/src/main/java/org/zowe/apiml/caching/service/infinispan/config/InfinispanConfig.java b/caching-service/src/main/java/org/zowe/apiml/caching/service/infinispan/config/InfinispanConfig.java index f82908a5a0..0f66feb596 100644 --- a/caching-service/src/main/java/org/zowe/apiml/caching/service/infinispan/config/InfinispanConfig.java +++ b/caching-service/src/main/java/org/zowe/apiml/caching/service/infinispan/config/InfinispanConfig.java @@ -43,11 +43,9 @@ import java.io.InputStream; import java.nio.file.Paths; import java.time.Duration; -import java.util.*; +import java.util.HashMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; -import java.util.stream.Collectors; -import java.util.stream.Stream; import static org.zowe.apiml.security.SecurityUtils.formatKeyringUrl; import static org.zowe.apiml.security.SecurityUtils.isKeyring; @@ -211,12 +209,11 @@ synchronized LazyCacheManager cacheManager(ResourceLoader resourceLoader, Applic System.setProperty("infinispan.ssl.trustStore", trustStore); System.setProperty("infinispan.ssl.trustStorePassword", trustStorePass); - var caches = Stream.of(CACHE_ZOWE, CACHE_ZOWE_INVALIDATED_TOKEN) - .collect(Collectors.toMap( cacheName -> cacheName, cacheName -> getDistributedCacheConfig())); + var caches = new HashMap(); + caches.put(CACHE_ZOWE, getDistributedCacheConfig()); + caches.put(CACHE_ZOWE_INVALIDATED_TOKEN, getDistributedCacheConfig()); if (applicationInfo.isModulith()) { - caches.put(CACHE_ZOWE, getDistributedCacheConfig()); - caches.put(CACHE_ZOWE_INVALIDATED_TOKEN, getDistributedCacheConfig()); caches.put("invalidatedJwtTokens", getDistributedCacheConfig()); // 1 minute to force zosmf tokens validation against zosmf for invalidated tokens diff --git a/integration-tests/src/test/java/org/zowe/apiml/functional/caching/JGroupStabilityTest.java b/integration-tests/src/test/java/org/zowe/apiml/functional/caching/JGroupStabilityTest.java index 2e50493198..b7663270c8 100644 --- a/integration-tests/src/test/java/org/zowe/apiml/functional/caching/JGroupStabilityTest.java +++ b/integration-tests/src/test/java/org/zowe/apiml/functional/caching/JGroupStabilityTest.java @@ -31,6 +31,7 @@ import javax.net.ssl.*; import java.io.*; import java.net.Socket; +import java.nio.file.Paths; import java.security.KeyManagementException; import java.security.NoSuchAlgorithmException; import java.security.SecureRandom; @@ -81,8 +82,8 @@ void givenTwoInstances_whenOneHasADelay_thenClusterIsRebuilt(boolean isModulith, try { cachingServices.forEach(CachingService::start); await() - .pollDelay(10, TimeUnit.SECONDS) - .timeout(2, TimeUnit.MINUTES) + .pollDelay(20, TimeUnit.SECONDS) + .timeout(isModulith ? 8 : 2, TimeUnit.MINUTES) .until(() -> cachingServices.stream().allMatch(CachingService::isUp)); cachingServices.get(0).pause(); @@ -167,20 +168,25 @@ static class CachingService { public void start() { int basePort = BASE_PORTS[index]; - log.info("Starting caching service on port {}", basePort); + String service = isModulith ? "apiml" : "caching-service"; + + log.info("Starting {} on ports based on {}", service, basePort); var env = new HashMap(); - env.put("ZWE_haInstance_id", "localhost" + String.valueOf(basePort).charAt(0)); + env.put("ZWE_configs_debug", "true"); + env.put("ZWE_haInstance_id", "localhost_" + (isModulith ? "Single" : "Multi") + "_" + (isAttls ? "Attls" : "NativeTls") + "_" + (index + 1)); env.put("APIML_ENABLED", isModulith ? "true" : "false"); - env.put("logbackService", "ZWEACS" + (index + 1)); - env.put("LAUNCH_COMPONENT", "caching-service/build/libs"); + env.put("logbackService", (isModulith ? "ZWEGW" : "ZWEACS") + (index + 1)); + env.put("LAUNCH_COMPONENT", service + "/build/libs"); env.put("ZWE_configs_port", String.valueOf(basePort + 25)); env.put("ZWE_configs_storage_infinispan_jgroups_port", String.valueOf(basePort + 600)); - env.put("ZWE_configs_storage_infinispan_jgroups_keyExchange_port", String.valueOf(BASE_PORTS[0] + 601)); + env.put("ZWE_configs_storage_infinispan_jgroups_host", "localhost"); + env.put("ZWE_configs_storage_infinispan_jgroups_keyExchange_port", String.valueOf(basePort + 601)); env.put("ZWE_configs_storage_infinispan_initialHosts", Arrays.stream(BASE_PORTS).mapToObj(bp -> "localhost[" + (bp + 600) + "]").collect(Collectors.joining(","))); env.put("ZWE_configs_storage_mode", "infinispan"); + env.put("ZWE_zowe_workspaceDirectory", Paths.get(".").toAbsolutePath().normalize().toString()); env.put("ZWE_zowe_certificate_keystore_file", "keystore/localhost/localhost.keystore.p12"); env.put("ZWE_zowe_certificate_keystore_password", "password"); @@ -192,9 +198,16 @@ public void start() { env.put("ZWE_configs_apiml_health_protected", "false"); env.put("attlsEnabledOnInfinispanTest", isAttls ? "true" : "false"); - env.put("ZWE_zowe_network_client_tls_attls", isAttls ? "true" : "false"); - ProcessBuilder builder = new ProcessBuilder("caching-service-package/src/main/resources/bin/start.sh"); + if (isModulith) { + env.put("CMMN_LB", "build/libs/api-layer-lite-lib-all.jar"); + env.put("ZWE_configs_internal_discovery_port", String.valueOf(basePort + 1)); + env.put("ZWE_configs_apiml_security_authorization_provider", "dummy"); + env.put("ZWE_components_gateway_apiml_security_auth_provider", "saf"); + env.put("ZWE_DISCOVERY_SERVICES_LIST", Arrays.stream(BASE_PORTS).mapToObj(bp -> "https://localhost:" + (bp + 1) + "/eureka/").collect(Collectors.joining(","))); + } + + ProcessBuilder builder = new ProcessBuilder(service + "-package/src/main/resources/bin/start.sh"); builder.environment().putAll(env); File binFolder = new File("../"); @@ -229,6 +242,7 @@ void readLogs(Process process, Consumer onPid) { void issue(String... parts) { ProcessBuilder builder = new ProcessBuilder(parts); + builder.redirectErrorStream(true); try { var process = builder.start(); readLogs(process, pid -> { @@ -263,7 +277,7 @@ public void kill() { public boolean isUp() { int basePort = BASE_PORTS[index]; - HttpGet request = new HttpGet("https://localhost:" + (basePort + 25) + "/cachingservice/application/health"); + HttpGet request = new HttpGet("https://localhost:" + (basePort + 25) + (isModulith ? "" : "/cachingservice") + "/application/health"); request.addHeader(HttpHeaders.ACCEPT, MediaType.APPLICATION_JSON_VALUE); try (CloseableHttpClient client = HttpClients.custom().setSSLContext(ignoreSslContext()).setSSLHostnameVerifier(new NoopHostnameVerifier()).build()) { final HttpResponse response = client.execute(request); @@ -271,8 +285,12 @@ public boolean isUp() { log.trace("URI: {}, JsonResponse is {}", request.getURI().toString(), jsonResponse); if (StringUtils.isNotEmpty(jsonResponse)) { - var status = JsonPath.parse(jsonResponse).read("components.caching.details.infinispan.cluster.status", String.class); - return "UP".equals(status); + var status = JsonPath.parse(jsonResponse).read(isModulith ? "components.infinispan.status" : "components.caching.details.infinispan.cluster.status", String.class); + boolean isUp = "UP".equals(status); + if (!isUp) { + log.warn("URI: {}, JsonResponse is {}", request.getURI().toString(), jsonResponse); + } + return isUp; } return false; } catch (Exception e) { diff --git a/integration-tests/src/test/java/org/zowe/apiml/startup/impl/ApiMediationLayerStartupChecker.java b/integration-tests/src/test/java/org/zowe/apiml/startup/impl/ApiMediationLayerStartupChecker.java index 0c8c1a2e39..8ac60c2e02 100644 --- a/integration-tests/src/test/java/org/zowe/apiml/startup/impl/ApiMediationLayerStartupChecker.java +++ b/integration-tests/src/test/java/org/zowe/apiml/startup/impl/ApiMediationLayerStartupChecker.java @@ -14,6 +14,7 @@ import com.jayway.jsonpath.JsonPath; import io.netty.handler.codec.http.HttpHeaderValues; import lombok.Data; +import lombok.EqualsAndHashCode; import lombok.extern.slf4j.Slf4j; import net.minidev.json.JSONArray; import org.apache.commons.lang3.StringUtils; @@ -21,8 +22,8 @@ import org.apache.http.HttpHeaders; import org.apache.http.HttpResponse; import org.apache.http.HttpStatus; -import org.apache.http.conn.ssl.NoopHostnameVerifier; import org.apache.http.client.methods.HttpGet; +import org.apache.http.conn.ssl.NoopHostnameVerifier; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; import org.apache.http.util.EntityUtils; @@ -74,6 +75,7 @@ public void waitUntilReady() { } @Data + @EqualsAndHashCode(of = {"hostname", "port", "serviceId"}) private static class Instance { private final String scheme; @@ -199,6 +201,7 @@ private ApimlInstance(Predicate instanceMatcher) { } return i; }) + .distinct() .filter(instanceMatcher) .collect(Collectors.toList()); } diff --git a/zaas-service/src/main/java/org/zowe/apiml/zaas/security/service/AuthenticationService.java b/zaas-service/src/main/java/org/zowe/apiml/zaas/security/service/AuthenticationService.java index 99a3a2d317..2fc155c3ff 100644 --- a/zaas-service/src/main/java/org/zowe/apiml/zaas/security/service/AuthenticationService.java +++ b/zaas-service/src/main/java/org/zowe/apiml/zaas/security/service/AuthenticationService.java @@ -51,12 +51,7 @@ import org.zowe.apiml.zaas.security.service.zosmf.ZosmfService; import java.text.ParseException; -import java.util.Arrays; -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; -import java.util.Set; +import java.util.*; import static org.zowe.apiml.security.common.util.JwtUtils.getJwtClaims; import static org.zowe.apiml.security.common.util.JwtUtils.handleJwtParserException; @@ -89,14 +84,34 @@ public class AuthenticationService { private final CacheManager cacheManager; private final CacheUtils cacheUtils; private boolean isModulithMode; - private Cache validatedJwtTokensCache; - private Cache invalidatedJwtTokensCache; + private volatile Cache validatedJwtTokensCache; + private volatile Cache invalidatedJwtTokensCache; @PostConstruct public void afterPropertiesSet() { isModulithMode = applicationContext.containsBean("modulithConfig"); - validatedJwtTokensCache = cacheManager.getCache(CACHE_VALIDATED_JWT_TOKENS); - invalidatedJwtTokensCache = cacheManager.getCache(CACHE_INVALIDATED_JWT_TOKENS); + } + + private Cache getValidatedJwtTokensCache() { + if (validatedJwtTokensCache == null) { + synchronized (AuthenticationService.class) { + if (validatedJwtTokensCache == null) { + validatedJwtTokensCache = cacheManager.getCache(CACHE_VALIDATED_JWT_TOKENS); + } + } + } + return validatedJwtTokensCache; + } + + private Cache getInvalidatedJwtTokensCache() { + if (invalidatedJwtTokensCache == null) { + synchronized (AuthenticationService.class) { + if (invalidatedJwtTokensCache == null) { + invalidatedJwtTokensCache = cacheManager.getCache(CACHE_INVALIDATED_JWT_TOKENS); + } + } + } + return invalidatedJwtTokensCache; } /** @@ -229,18 +244,21 @@ private Boolean doInvalidate(String jwtToken, boolean distribute, Application ap } private void putValidationCache(String jwtToken, TokenAuthentication tokenAuthentication) { + var validatedJwtTokensCache = getValidatedJwtTokensCache(); if (jwtToken != null && validatedJwtTokensCache != null) { validatedJwtTokensCache.put(jwtToken, tokenAuthentication); } } private void evictValidationCache(String jwtToken) { + var validatedJwtTokensCache = getValidatedJwtTokensCache(); if (validatedJwtTokensCache != null) { validatedJwtTokensCache.evict(jwtToken); } } private void putInvalidatedCache(String jwtToken) { + var invalidatedJwtTokensCache = getInvalidatedJwtTokensCache(); if (invalidatedJwtTokensCache != null) { invalidatedJwtTokensCache.put(jwtToken, Boolean.TRUE); } @@ -307,6 +325,7 @@ private boolean invalidateTokenOnAnotherInstance(String jwtToken, Application ap * @return true - token is invalidated, otherwise token is still valid */ public boolean isInvalidated(String jwtToken) { + var invalidatedJwtTokensCache = getInvalidatedJwtTokensCache(); if (invalidatedJwtTokensCache == null) { return false; } @@ -361,6 +380,7 @@ public TokenAuthentication validateJwtToken(String jwtToken) { throw new TokenNotValidException("Token ...%s was invalidated.".formatted(StringUtils.right(jwtToken, 15))); } + var validatedJwtTokensCache = getValidatedJwtTokensCache(); if (validatedJwtTokensCache != null) { Cache.ValueWrapper cached = validatedJwtTokensCache.get(jwtToken); if (cached != null) { diff --git a/zaas-service/src/test/java/org/zowe/apiml/zaas/security/service/AuthenticationServiceTest.java b/zaas-service/src/test/java/org/zowe/apiml/zaas/security/service/AuthenticationServiceTest.java index 6eeffda6c4..e2fe08c42c 100644 --- a/zaas-service/src/test/java/org/zowe/apiml/zaas/security/service/AuthenticationServiceTest.java +++ b/zaas-service/src/test/java/org/zowe/apiml/zaas/security/service/AuthenticationServiceTest.java @@ -127,8 +127,8 @@ void setup() { authConfigurationProperties = new AuthConfigurationProperties(); authConfigurationProperties.getZosmf().setServiceId(ZOSMF); - when(cacheManager.getCache("validatedJwtTokens")).thenReturn(validatedJwtTokensCache); - when(cacheManager.getCache("invalidatedJwtTokens")).thenReturn(invalidatedJwtTokensCache); + lenient().when(cacheManager.getCache("validatedJwtTokens")).thenReturn(validatedJwtTokensCache); + lenient().when(cacheManager.getCache("invalidatedJwtTokens")).thenReturn(invalidatedJwtTokensCache); authService = new AuthenticationService( applicationContext, authConfigurationProperties, jwtSecurityInitializer,