chore: Fix caching service stability test for Modulith

.github/workflows/integration-tests.yml

@@ -997,7 +997,6 @@ jobs:
             caching-service:
                 image: ghcr.io/balhar-jakub/caching-service:${{ github.run_id }}-${{ github.run_number }}
                 env:
-                    ZWE_CACHING_SERVICE_PERSISTENT: 'infinispan'
                     CACHING_STORAGE_MODE: "infinispan"
                     JGROUPS_BIND_ADDRESS: "caching-service"
                     JGROUPS_BIND_PORT: "7600"
@@ -2164,7 +2163,6 @@ jobs:
             caching-service:
                 image: ghcr.io/balhar-jakub/caching-service:${{ github.run_id }}-${{ github.run_number }}
                 env:
-                    ZWE_CACHING_SERVICE_PERSISTENT: 'infinispan'
                     JGROUPS_BIND_PORT: "7600"
                     SERVER_SSL_KEYSTORETYPE: "PKCS12"
                     CACHING_STORAGE_INFINISPAN_PERSISTENCE_DATALOCATION: "data_replica"
@@ -2175,7 +2173,6 @@ jobs:
             caching-service-2:
                 image: ghcr.io/balhar-jakub/caching-service:${{ github.run_id }}-${{ github.run_number }}
                 env:
-                    ZWE_CACHING_SERVICE_PERSISTENT: 'infinispan'
                     JGROUPS_BIND_PORT: "7600"
                     SERVER_SSL_KEYSTORETYPE: "PKCS12"
                     CACHING_STORAGE_INFINISPAN_PERSISTENCE_DATALOCATION: "data"
@@ -2250,29 +2247,27 @@ jobs:
                     APIML_SECURITY_X509_ENABLED: true
                     APIML_SECURITY_SSL_NONSTRICTVERIFYSSLCERTIFICATESOFSERVICES: true
                     APIML_DISCOVERY_ALLPEERSURLS: https://apiml-2:10011/eureka,https://apiml:10011/eureka
-                    ZWE_CACHING_SERVICE_PERSISTENT: 'infinispan'
-                    JGROUPS_BIND_PORT: "7600"
-                    SERVER_SSL_KEYSTORETYPE: "PKCS12"
-                    CACHING_STORAGE_INFINISPAN_INITIALHOSTS: "apiml[7600],apiml-2[7600]"
+                    CACHING_STORAGE_INFINISPAN_NUMSEGMENTS: "16"
                     CACHING_STORAGE_MODE: "infinispan"
+                    CACHING_STORAGE_INFINISPAN_INITIALHOSTS: "apiml[7600],apiml-2[7600]"
+                    JGROUPS_BIND_PORT: "7600"
                     JGROUPS_BIND_ADDRESS: "apiml"
-                    CACHING_STORAGE_INFINISPAN_NUMSEGMENTS: "16"
+                    JGROUPS_KEYEXCHANGE_PORT: 7601
             apiml-2:
                 image: ghcr.io/balhar-jakub/apiml:${{ github.run_id }}-${{ github.run_number }}
                 env:
                     APIML_SECURITY_X509_ENABLED: true
                     APIML_GATEWAY_SERVICESTOLIMITREQUESTRATE: discoverableclient
                     APIML_SECURITY_SSL_NONSTRICTVERIFYSSLCERTIFICATESOFSERVICES: true
                     APIML_DISCOVERY_ALLPEERSURLS: https://apiml:10011/eureka,https://apiml-2:10011/eureka
-                    ZWE_CACHING_SERVICE_PERSISTENT: 'infinispan'
-                    JGROUPS_BIND_PORT: "7600"
-                    SERVER_SSL_KEYSTORETYPE: "PKCS12"
-                    CACHING_STORAGE_INFINISPAN_INITIALHOSTS: "apiml[7600],apiml-2[7600]"
-                    CACHING_STORAGE_MODE: "infinispan"
-                    JGROUPS_BIND_ADDRESS: "apiml-2"
                     APIML_SERVICE_HOSTNAME: "apiml-2"
                     logbackService: ZWEAGW2
                     CACHING_STORAGE_INFINISPAN_NUMSEGMENTS: "16"
+                    CACHING_STORAGE_MODE: "infinispan"
+                    CACHING_STORAGE_INFINISPAN_INITIALHOSTS: "apiml[7600],apiml-2[7600]"
+                    JGROUPS_BIND_PORT: "7600"
+                    JGROUPS_BIND_ADDRESS: "apiml-2"
+                    JGROUPS_KEYEXCHANGE_PORT: 7601
             api-catalog-services:
                 image: ghcr.io/balhar-jakub/api-catalog-services:${{ github.run_id }}-${{ github.run_number }}
                 volumes:

apiml/src/main/java/org/zowe/apiml/ApimlApplication.java

@@ -12,6 +12,7 @@
 
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.actuate.autoconfigure.logging.OpenTelemetryLoggingAutoConfiguration;
+import org.springframework.boot.actuate.autoconfigure.metrics.cache.CacheMetricsAutoConfiguration;
 import org.springframework.boot.actuate.autoconfigure.opentelemetry.OpenTelemetryAutoConfiguration;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.boot.autoconfigure.security.oauth2.client.reactive.ReactiveOAuth2ClientAutoConfiguration;
@@ -28,7 +29,8 @@
         ReactiveOAuth2ClientAutoConfiguration.class,
         OpenTelemetryAutoConfiguration.class,
         OpenTelemetryLoggingAutoConfiguration.class,
-        io.opentelemetry.instrumentation.spring.autoconfigure.OpenTelemetryAutoConfiguration.class
+        io.opentelemetry.instrumentation.spring.autoconfigure.OpenTelemetryAutoConfiguration.class,
+        CacheMetricsAutoConfiguration.class
     },
     scanBasePackages = {
         "org.zowe.apiml.filter",

apiml/src/main/resources/application.yml

@@ -346,7 +346,9 @@ logging:
         org.apache.tomcat.util.net: DEBUG
         org.apache.tomcat.util.net.jsse.JSSESupport: INFO
         org.ehcache: INFO
-        org.jgroups: DEBUG
+        org.jgroups: TRACE
+        org.jgroups.protocols: ERROR
+        org.infinispan: DEBUG
         org.springframework.context.support.[PostProcessorRegistrationDelegate$BeanPostProcessorChecker]: DEBUG
         org.springframework.security: TRACE
         org.springframework.security.web: TRACE

caching-service/src/main/java/org/zowe/apiml/caching/service/infinispan/config/InfinispanConfig.java

@@ -43,11 +43,9 @@
 import java.io.InputStream;
 import java.nio.file.Paths;
 import java.time.Duration;
-import java.util.*;
+import java.util.HashMap;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicReference;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
 
 import static org.zowe.apiml.security.SecurityUtils.formatKeyringUrl;
 import static org.zowe.apiml.security.SecurityUtils.isKeyring;
@@ -211,12 +209,11 @@ synchronized LazyCacheManager cacheManager(ResourceLoader resourceLoader, Applic
         System.setProperty("infinispan.ssl.trustStore", trustStore);
         System.setProperty("infinispan.ssl.trustStorePassword", trustStorePass);
 
-        var caches = Stream.of(CACHE_ZOWE, CACHE_ZOWE_INVALIDATED_TOKEN)
-            .collect(Collectors.toMap( cacheName -> cacheName, cacheName -> getDistributedCacheConfig()));
+        var caches = new HashMap<String, ConfigurationBuilder>();
+        caches.put(CACHE_ZOWE, getDistributedCacheConfig());
+        caches.put(CACHE_ZOWE_INVALIDATED_TOKEN, getDistributedCacheConfig());
 
         if (applicationInfo.isModulith()) {
-            caches.put(CACHE_ZOWE, getDistributedCacheConfig());
-            caches.put(CACHE_ZOWE_INVALIDATED_TOKEN, getDistributedCacheConfig());
             caches.put("invalidatedJwtTokens", getDistributedCacheConfig());
 
             // 1 minute to force zosmf tokens validation against zosmf for invalidated tokens

integration-tests/src/test/java/org/zowe/apiml/functional/caching/JGroupStabilityTest.java

@@ -31,6 +31,7 @@
 import javax.net.ssl.*;
 import java.io.*;
 import java.net.Socket;
+import java.nio.file.Paths;
 import java.security.KeyManagementException;
 import java.security.NoSuchAlgorithmException;
 import java.security.SecureRandom;
@@ -81,8 +82,8 @@ void givenTwoInstances_whenOneHasADelay_thenClusterIsRebuilt(boolean isModulith,
         try {
             cachingServices.forEach(CachingService::start);
             await()
-                .pollDelay(10, TimeUnit.SECONDS)
-                .timeout(2, TimeUnit.MINUTES)
+                .pollDelay(20, TimeUnit.SECONDS)
+                .timeout(isModulith ? 8 : 2, TimeUnit.MINUTES)
                 .until(() -> cachingServices.stream().allMatch(CachingService::isUp));
 
             cachingServices.get(0).pause();
@@ -167,20 +168,25 @@ static class CachingService {
 
         public void start() {
             int basePort = BASE_PORTS[index];
-            log.info("Starting caching service on port {}", basePort);
+            String service = isModulith ? "apiml" : "caching-service";
+
+            log.info("Starting {} on ports based on {}", service, basePort);
 
             var env = new HashMap<String, String>();
-            env.put("ZWE_haInstance_id", "localhost" + String.valueOf(basePort).charAt(0));
+            env.put("ZWE_configs_debug", "true");
+            env.put("ZWE_haInstance_id", "localhost_" + (isModulith ? "Single" : "Multi") + "_" + (isAttls ? "Attls" : "NativeTls") + "_" + (index + 1));
             env.put("APIML_ENABLED", isModulith ? "true" : "false");
-            env.put("logbackService", "ZWEACS" + (index + 1));
-            env.put("LAUNCH_COMPONENT", "caching-service/build/libs");
+            env.put("logbackService", (isModulith ? "ZWEGW" : "ZWEACS") + (index + 1));
+            env.put("LAUNCH_COMPONENT", service + "/build/libs");
 
             env.put("ZWE_configs_port", String.valueOf(basePort + 25));
 
             env.put("ZWE_configs_storage_infinispan_jgroups_port", String.valueOf(basePort + 600));
-            env.put("ZWE_configs_storage_infinispan_jgroups_keyExchange_port", String.valueOf(BASE_PORTS[0] + 601));
+            env.put("ZWE_configs_storage_infinispan_jgroups_host", "localhost");
+            env.put("ZWE_configs_storage_infinispan_jgroups_keyExchange_port", String.valueOf(basePort + 601));
             env.put("ZWE_configs_storage_infinispan_initialHosts", Arrays.stream(BASE_PORTS).mapToObj(bp -> "localhost[" + (bp + 600) + "]").collect(Collectors.joining(",")));
             env.put("ZWE_configs_storage_mode", "infinispan");
+            env.put("ZWE_zowe_workspaceDirectory", Paths.get(".").toAbsolutePath().normalize().toString());
 
             env.put("ZWE_zowe_certificate_keystore_file", "keystore/localhost/localhost.keystore.p12");
             env.put("ZWE_zowe_certificate_keystore_password", "password");
@@ -192,9 +198,16 @@ public void start() {
 
             env.put("ZWE_configs_apiml_health_protected", "false");
             env.put("attlsEnabledOnInfinispanTest", isAttls ? "true" : "false");
-            env.put("ZWE_zowe_network_client_tls_attls", isAttls ? "true" : "false");
 
-            ProcessBuilder builder = new ProcessBuilder("caching-service-package/src/main/resources/bin/start.sh");
+            if (isModulith) {
+                env.put("CMMN_LB", "build/libs/api-layer-lite-lib-all.jar");
+                env.put("ZWE_configs_internal_discovery_port", String.valueOf(basePort + 1));
+                env.put("ZWE_configs_apiml_security_authorization_provider", "dummy");
+                env.put("ZWE_components_gateway_apiml_security_auth_provider", "saf");
+                env.put("ZWE_DISCOVERY_SERVICES_LIST", Arrays.stream(BASE_PORTS).mapToObj(bp -> "https://localhost:" + (bp + 1) + "/eureka/").collect(Collectors.joining(",")));
+            }
+
+            ProcessBuilder builder = new ProcessBuilder(service + "-package/src/main/resources/bin/start.sh");
             builder.environment().putAll(env);
 
             File binFolder = new File("../");
@@ -229,6 +242,7 @@ void readLogs(Process process, Consumer<String> onPid) {
 
         void issue(String... parts) {
             ProcessBuilder builder = new ProcessBuilder(parts);
+            builder.redirectErrorStream(true);
             try {
                 var process = builder.start();
                 readLogs(process, pid -> {
@@ -263,16 +277,20 @@ public void kill() {
 
         public boolean isUp() {
             int basePort = BASE_PORTS[index];
-            HttpGet request = new HttpGet("https://localhost:" + (basePort + 25) + "/cachingservice/application/health");
+            HttpGet request = new HttpGet("https://localhost:" + (basePort + 25) + (isModulith ? "" : "/cachingservice") + "/application/health");
             request.addHeader(HttpHeaders.ACCEPT, MediaType.APPLICATION_JSON_VALUE);
             try (CloseableHttpClient client = HttpClients.custom().setSSLContext(ignoreSslContext()).setSSLHostnameVerifier(new NoopHostnameVerifier()).build()) {
                 final HttpResponse response = client.execute(request);
                 final String jsonResponse = EntityUtils.toString(response.getEntity());
                 log.trace("URI: {}, JsonResponse is {}", request.getURI().toString(), jsonResponse);
 
                 if (StringUtils.isNotEmpty(jsonResponse)) {
-                    var status = JsonPath.parse(jsonResponse).read("components.caching.details.infinispan.cluster.status", String.class);
-                    return "UP".equals(status);
+                    var status = JsonPath.parse(jsonResponse).read(isModulith ? "components.infinispan.status" : "components.caching.details.infinispan.cluster.status", String.class);
+                    boolean isUp = "UP".equals(status);
+                    if (!isUp) {
+                        log.warn("URI: {}, JsonResponse is {}", request.getURI().toString(), jsonResponse);
+                    }
+                    return isUp;
                 }
                 return false;
             } catch (Exception e) {

integration-tests/src/test/java/org/zowe/apiml/startup/impl/ApiMediationLayerStartupChecker.java

@@ -14,15 +14,16 @@
 import com.jayway.jsonpath.JsonPath;
 import io.netty.handler.codec.http.HttpHeaderValues;
 import lombok.Data;
+import lombok.EqualsAndHashCode;
 import lombok.extern.slf4j.Slf4j;
 import net.minidev.json.JSONArray;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.Strings;
 import org.apache.http.HttpHeaders;
 import org.apache.http.HttpResponse;
 import org.apache.http.HttpStatus;
-import org.apache.http.conn.ssl.NoopHostnameVerifier;
 import org.apache.http.client.methods.HttpGet;
+import org.apache.http.conn.ssl.NoopHostnameVerifier;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClients;
 import org.apache.http.util.EntityUtils;
@@ -74,6 +75,7 @@ public void waitUntilReady() {
     }
 
     @Data
+    @EqualsAndHashCode(of = {"hostname", "port", "serviceId"})
     private static class Instance {
 
         private final String scheme;
@@ -199,6 +201,7 @@ private ApimlInstance(Predicate<Instance> instanceMatcher) {
                     }
                     return i;
                 })
+                .distinct()
                 .filter(instanceMatcher)
                 .collect(Collectors.toList());
         }

zaas-service/src/main/java/org/zowe/apiml/zaas/security/service/AuthenticationService.java

@@ -51,12 +51,7 @@
 import org.zowe.apiml.zaas.security.service.zosmf.ZosmfService;
 
 import java.text.ParseException;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
+import java.util.*;
 
 import static org.zowe.apiml.security.common.util.JwtUtils.getJwtClaims;
 import static org.zowe.apiml.security.common.util.JwtUtils.handleJwtParserException;
@@ -89,14 +84,34 @@
     private final CacheManager cacheManager;
     private final CacheUtils cacheUtils;
     private boolean isModulithMode;
-    private Cache validatedJwtTokensCache;
-    private Cache invalidatedJwtTokensCache;
+    private volatile Cache validatedJwtTokensCache;
+    private volatile Cache invalidatedJwtTokensCache;
 
     @PostConstruct
     public void afterPropertiesSet() {
         isModulithMode = applicationContext.containsBean("modulithConfig");
-        validatedJwtTokensCache = cacheManager.getCache(CACHE_VALIDATED_JWT_TOKENS);
-        invalidatedJwtTokensCache = cacheManager.getCache(CACHE_INVALIDATED_JWT_TOKENS);
+    }
+
+    private Cache getValidatedJwtTokensCache() {
+        if (validatedJwtTokensCache == null) {
+            synchronized (AuthenticationService.class) {
+                if (validatedJwtTokensCache == null) {
+                    validatedJwtTokensCache = cacheManager.getCache(CACHE_VALIDATED_JWT_TOKENS);
+                }
+            }
+        }
+        return validatedJwtTokensCache;
+    }
+
+    private Cache getInvalidatedJwtTokensCache() {
+        if (invalidatedJwtTokensCache == null) {
+            synchronized (AuthenticationService.class) {
+                if (invalidatedJwtTokensCache == null) {
+                    invalidatedJwtTokensCache = cacheManager.getCache(CACHE_INVALIDATED_JWT_TOKENS);
+                }
+            }
+        }
+        return invalidatedJwtTokensCache;
     }
 
     /**
@@ -229,18 +244,21 @@
     }
 
     private void putValidationCache(String jwtToken, TokenAuthentication tokenAuthentication) {
+        var validatedJwtTokensCache = getValidatedJwtTokensCache();
         if (jwtToken != null && validatedJwtTokensCache != null) {
             validatedJwtTokensCache.put(jwtToken, tokenAuthentication);
         }
     }
 
     private void evictValidationCache(String jwtToken) {
+        var validatedJwtTokensCache = getValidatedJwtTokensCache();
         if (validatedJwtTokensCache != null) {
             validatedJwtTokensCache.evict(jwtToken);
         }
     }
 
     private void putInvalidatedCache(String jwtToken) {
+        var invalidatedJwtTokensCache = getInvalidatedJwtTokensCache();
         if (invalidatedJwtTokensCache != null) {
             invalidatedJwtTokensCache.put(jwtToken, Boolean.TRUE);
         }
@@ -307,6 +325,7 @@
      * @return true - token is invalidated, otherwise token is still valid
      */
     public boolean isInvalidated(String jwtToken) {
+        var invalidatedJwtTokensCache = getInvalidatedJwtTokensCache();
         if (invalidatedJwtTokensCache == null) {
             return false;
         }
@@ -361,6 +380,7 @@
             throw new TokenNotValidException("Token ...%s was invalidated.".formatted(StringUtils.right(jwtToken, 15)));
         }
 
+        var validatedJwtTokensCache = getValidatedJwtTokensCache();
         if (validatedJwtTokensCache != null) {
             Cache.ValueWrapper cached = validatedJwtTokensCache.get(jwtToken);
             if (cached != null) {

zaas-service/src/test/java/org/zowe/apiml/zaas/security/service/AuthenticationServiceTest.java

@@ -127,8 +127,8 @@ void setup() {
         authConfigurationProperties = new AuthConfigurationProperties();
         authConfigurationProperties.getZosmf().setServiceId(ZOSMF);
 
-        when(cacheManager.getCache("validatedJwtTokens")).thenReturn(validatedJwtTokensCache);
-        when(cacheManager.getCache("invalidatedJwtTokens")).thenReturn(invalidatedJwtTokensCache);
+        lenient().when(cacheManager.getCache("validatedJwtTokens")).thenReturn(validatedJwtTokensCache);
+        lenient().when(cacheManager.getCache("invalidatedJwtTokens")).thenReturn(invalidatedJwtTokensCache);
 
         authService = new AuthenticationService(
             applicationContext, authConfigurationProperties, jwtSecurityInitializer,