fix: Lazy initialization of CacheManager

.github/workflows/integration-tests.yml

@@ -997,7 +997,6 @@ jobs:
             caching-service:
                 image: ghcr.io/balhar-jakub/caching-service:${{ github.run_id }}-${{ github.run_number }}
                 env:
-                    ZWE_CACHING_SERVICE_PERSISTENT: 'infinispan'
                     CACHING_STORAGE_MODE: "infinispan"
                     JGROUPS_BIND_ADDRESS: "caching-service"
                     JGROUPS_BIND_PORT: "7600"
@@ -2164,7 +2163,6 @@ jobs:
             caching-service:
                 image: ghcr.io/balhar-jakub/caching-service:${{ github.run_id }}-${{ github.run_number }}
                 env:
-                    ZWE_CACHING_SERVICE_PERSISTENT: 'infinispan'
                     JGROUPS_BIND_PORT: "7600"
                     SERVER_SSL_KEYSTORETYPE: "PKCS12"
                     CACHING_STORAGE_INFINISPAN_PERSISTENCE_DATALOCATION: "data_replica"
@@ -2175,7 +2173,6 @@ jobs:
             caching-service-2:
                 image: ghcr.io/balhar-jakub/caching-service:${{ github.run_id }}-${{ github.run_number }}
                 env:
-                    ZWE_CACHING_SERVICE_PERSISTENT: 'infinispan'
                     JGROUPS_BIND_PORT: "7600"
                     SERVER_SSL_KEYSTORETYPE: "PKCS12"
                     CACHING_STORAGE_INFINISPAN_PERSISTENCE_DATALOCATION: "data"
@@ -2250,29 +2247,29 @@ jobs:
                     APIML_SECURITY_X509_ENABLED: true
                     APIML_SECURITY_SSL_NONSTRICTVERIFYSSLCERTIFICATESOFSERVICES: true
                     APIML_DISCOVERY_ALLPEERSURLS: https://apiml-2:10011/eureka,https://apiml:10011/eureka
-                    ZWE_CACHING_SERVICE_PERSISTENT: 'infinispan'
-                    JGROUPS_BIND_PORT: "7600"
                     SERVER_SSL_KEYSTORETYPE: "PKCS12"
-                    CACHING_STORAGE_INFINISPAN_INITIALHOSTS: "apiml[7600],apiml-2[7600]"
+                    CACHING_STORAGE_INFINISPAN_NUMSEGMENTS: "16"
                     CACHING_STORAGE_MODE: "infinispan"
+                    CACHING_STORAGE_INFINISPAN_INITIALHOSTS: "apiml[7600],apiml-2[7600]"
+                    JGROUPS_BIND_PORT: "7600"
                     JGROUPS_BIND_ADDRESS: "apiml"
-                    CACHING_STORAGE_INFINISPAN_NUMSEGMENTS: "16"
+                    JGROUPS_KEYEXCHANGE_PORT: 7601
             apiml-2:
                 image: ghcr.io/balhar-jakub/apiml:${{ github.run_id }}-${{ github.run_number }}
                 env:
                     APIML_SECURITY_X509_ENABLED: true
                     APIML_GATEWAY_SERVICESTOLIMITREQUESTRATE: discoverableclient
                     APIML_SECURITY_SSL_NONSTRICTVERIFYSSLCERTIFICATESOFSERVICES: true
                     APIML_DISCOVERY_ALLPEERSURLS: https://apiml:10011/eureka,https://apiml-2:10011/eureka
-                    ZWE_CACHING_SERVICE_PERSISTENT: 'infinispan'
-                    JGROUPS_BIND_PORT: "7600"
                     SERVER_SSL_KEYSTORETYPE: "PKCS12"
-                    CACHING_STORAGE_INFINISPAN_INITIALHOSTS: "apiml[7600],apiml-2[7600]"
-                    CACHING_STORAGE_MODE: "infinispan"
-                    JGROUPS_BIND_ADDRESS: "apiml-2"
                     APIML_SERVICE_HOSTNAME: "apiml-2"
                     logbackService: ZWEAGW2
                     CACHING_STORAGE_INFINISPAN_NUMSEGMENTS: "16"
+                    CACHING_STORAGE_MODE: "infinispan"
+                    CACHING_STORAGE_INFINISPAN_INITIALHOSTS: "apiml[7600],apiml-2[7600]"
+                    JGROUPS_BIND_PORT: "7600"
+                    JGROUPS_BIND_ADDRESS: "apiml-2"
+                    JGROUPS_KEYEXCHANGE_PORT: 7601
             api-catalog-services:
                 image: ghcr.io/balhar-jakub/api-catalog-services:${{ github.run_id }}-${{ github.run_number }}
                 volumes:
@@ -2296,7 +2293,8 @@ jobs:
             -   name: Run Startup Check
                 if: always()
                 run: >
-                    ./gradlew runStartUpCheck --info -Denvironment.config=-docker-modulith
+                    ./gradlew runStartUpCheck --info -Denvironment.config=-docker-modulith-ha
+                    -Dgateway.instances=2 -Ddiscovery.instances=2 -Dcaching.instances=2 -Dapicatalog.instances=2
                     -Denvironment.modulith=true -Ddiscoverableclient.instances=1 -DcentralGateway.instances=0
                     -Partifactory_user=$ARTIFACTORY_USERNAME -Partifactory_password=$ARTIFACTORY_PASSWORD
                 env:

apiml/src/main/java/org/zowe/apiml/ApimlApplication.java

@@ -12,6 +12,7 @@
 
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.actuate.autoconfigure.logging.OpenTelemetryLoggingAutoConfiguration;
+import org.springframework.boot.actuate.autoconfigure.metrics.cache.CacheMetricsAutoConfiguration;
 import org.springframework.boot.actuate.autoconfigure.opentelemetry.OpenTelemetryAutoConfiguration;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.boot.autoconfigure.security.oauth2.client.reactive.ReactiveOAuth2ClientAutoConfiguration;
@@ -28,7 +29,8 @@
         ReactiveOAuth2ClientAutoConfiguration.class,
         OpenTelemetryAutoConfiguration.class,
         OpenTelemetryLoggingAutoConfiguration.class,
-        io.opentelemetry.instrumentation.spring.autoconfigure.OpenTelemetryAutoConfiguration.class
+        io.opentelemetry.instrumentation.spring.autoconfigure.OpenTelemetryAutoConfiguration.class,
+        CacheMetricsAutoConfiguration.class
     },
     scanBasePackages = {
         "org.zowe.apiml.filter",

caching-service/src/main/java/org/zowe/apiml/caching/service/infinispan/config/InfinispanConfig.java

@@ -43,11 +43,9 @@
 import java.io.InputStream;
 import java.nio.file.Paths;
 import java.time.Duration;
-import java.util.*;
+import java.util.HashMap;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicReference;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
 
 import static org.zowe.apiml.security.SecurityUtils.formatKeyringUrl;
 import static org.zowe.apiml.security.SecurityUtils.isKeyring;
@@ -211,12 +209,11 @@ synchronized LazyCacheManager cacheManager(ResourceLoader resourceLoader, Applic
         System.setProperty("infinispan.ssl.trustStore", trustStore);
         System.setProperty("infinispan.ssl.trustStorePassword", trustStorePass);
 
-        var caches = Stream.of(CACHE_ZOWE, CACHE_ZOWE_INVALIDATED_TOKEN)
-            .collect(Collectors.toMap( cacheName -> cacheName, cacheName -> getDistributedCacheConfig()));
+        var caches = new HashMap<String, ConfigurationBuilder>();
+        caches.put(CACHE_ZOWE, getDistributedCacheConfig());
+        caches.put(CACHE_ZOWE_INVALIDATED_TOKEN, getDistributedCacheConfig());
 
         if (applicationInfo.isModulith()) {
-            caches.put(CACHE_ZOWE, getDistributedCacheConfig());
-            caches.put(CACHE_ZOWE_INVALIDATED_TOKEN, getDistributedCacheConfig());
             caches.put("invalidatedJwtTokens", getDistributedCacheConfig());
 
             // 1 minute to force zosmf tokens validation against zosmf for invalidated tokens

zaas-service/src/main/java/org/zowe/apiml/zaas/security/service/AuthenticationService.java

@@ -51,12 +51,8 @@
 import org.zowe.apiml.zaas.security.service.zosmf.ZosmfService;
 
 import java.text.ParseException;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
+import java.util.*;
+import java.util.concurrent.atomic.AtomicReference;
 
 import static org.zowe.apiml.security.common.util.JwtUtils.getJwtClaims;
 import static org.zowe.apiml.security.common.util.JwtUtils.handleJwtParserException;
@@ -89,14 +85,38 @@ public class AuthenticationService {
     private final CacheManager cacheManager;
     private final CacheUtils cacheUtils;
     private boolean isModulithMode;
-    private Cache validatedJwtTokensCache;
-    private Cache invalidatedJwtTokensCache;
+    private final AtomicReference<Cache> validatedJwtTokensCache = new AtomicReference<>();
+    private final AtomicReference<Cache> invalidatedJwtTokensCache = new AtomicReference<>();
 
     @PostConstruct
     public void afterPropertiesSet() {
         isModulithMode = applicationContext.containsBean("modulithConfig");
-        validatedJwtTokensCache = cacheManager.getCache(CACHE_VALIDATED_JWT_TOKENS);
-        invalidatedJwtTokensCache = cacheManager.getCache(CACHE_INVALIDATED_JWT_TOKENS);
+    }
+
+    private Cache getValidatedJwtTokensCache() {
+        var cacheValidatedJwtTokensCache = this.validatedJwtTokensCache.get();
+        if (cacheValidatedJwtTokensCache == null) {
+            synchronized (validatedJwtTokensCache) {
+                cacheValidatedJwtTokensCache = validatedJwtTokensCache.get();
+                if (cacheValidatedJwtTokensCache == null) {
+                    validatedJwtTokensCache.set(cacheManager.getCache(CACHE_VALIDATED_JWT_TOKENS));
+                }
+            }
+        }
+        return validatedJwtTokensCache.get();
+    }
+
+    private Cache getInvalidatedJwtTokensCache() {
+        var cacheInvalidatedJwtTokensCache = this.invalidatedJwtTokensCache.get();
+        if (cacheInvalidatedJwtTokensCache == null) {
+            synchronized (invalidatedJwtTokensCache) {
+                cacheInvalidatedJwtTokensCache = this.invalidatedJwtTokensCache.get();
+                if (cacheInvalidatedJwtTokensCache == null) {
+                    invalidatedJwtTokensCache.set(cacheManager.getCache(CACHE_INVALIDATED_JWT_TOKENS));
+                }
+            }
+        }
+        return invalidatedJwtTokensCache.get();
     }
 
     /**
@@ -229,20 +249,23 @@ private Boolean doInvalidate(String jwtToken, boolean distribute, Application ap
     }
 
     private void putValidationCache(String jwtToken, TokenAuthentication tokenAuthentication) {
-        if (jwtToken != null && validatedJwtTokensCache != null) {
-            validatedJwtTokensCache.put(jwtToken, tokenAuthentication);
+        var cacheValidatedJwtTokens = getValidatedJwtTokensCache();
+        if (jwtToken != null && cacheValidatedJwtTokens != null) {
+            cacheValidatedJwtTokens.put(jwtToken, tokenAuthentication);
         }
     }
 
     private void evictValidationCache(String jwtToken) {
-        if (validatedJwtTokensCache != null) {
-            validatedJwtTokensCache.evict(jwtToken);
+        var cacheValidatedJwtTokens = getValidatedJwtTokensCache();
+        if (cacheValidatedJwtTokens != null) {
+            cacheValidatedJwtTokens.evict(jwtToken);
         }
     }
 
     private void putInvalidatedCache(String jwtToken) {
-        if (invalidatedJwtTokensCache != null) {
-            invalidatedJwtTokensCache.put(jwtToken, Boolean.TRUE);
+        var cacheInvalidatedJwtTokens = getInvalidatedJwtTokensCache();
+        if (cacheInvalidatedJwtTokens != null) {
+            cacheInvalidatedJwtTokens.put(jwtToken, Boolean.TRUE);
         }
     }
 
@@ -307,10 +330,11 @@ private boolean invalidateTokenOnAnotherInstance(String jwtToken, Application ap
      * @return true - token is invalidated, otherwise token is still valid
      */
     public boolean isInvalidated(String jwtToken) {
-        if (invalidatedJwtTokensCache == null) {
+        var cacheInvalidatedJwtTokens = getInvalidatedJwtTokensCache();
+        if (cacheInvalidatedJwtTokens == null) {
             return false;
         }
-        Cache.ValueWrapper wrapper = invalidatedJwtTokensCache.get(jwtToken);
+        Cache.ValueWrapper wrapper = cacheInvalidatedJwtTokens.get(jwtToken);
         boolean result = wrapper != null && Boolean.TRUE.equals(wrapper.get());
         log.debug("Token invalidation check for ...{}: {}", StringUtils.right(jwtToken, 15), result);
         return result;
@@ -361,8 +385,9 @@ public TokenAuthentication validateJwtToken(String jwtToken) {
             throw new TokenNotValidException("Token ...%s was invalidated.".formatted(StringUtils.right(jwtToken, 15)));
         }
 
-        if (validatedJwtTokensCache != null) {
-            Cache.ValueWrapper cached = validatedJwtTokensCache.get(jwtToken);
+        var cacheValidatedJwtTokens = getValidatedJwtTokensCache();
+        if (cacheValidatedJwtTokens != null) {
+            Cache.ValueWrapper cached = cacheValidatedJwtTokens.get(jwtToken);
             if (cached != null) {
                 var tokenAuthentication = (TokenAuthentication) cached.get();
                 log.debug("JWT ...{} found in the cache. Is authenticated: {}", StringUtils.right(jwtToken, 15), tokenAuthentication.isAuthenticated());

zaas-service/src/test/java/org/zowe/apiml/zaas/security/service/AuthenticationServiceTest.java

@@ -127,8 +127,8 @@ void setup() {
         authConfigurationProperties = new AuthConfigurationProperties();
         authConfigurationProperties.getZosmf().setServiceId(ZOSMF);
 
-        when(cacheManager.getCache("validatedJwtTokens")).thenReturn(validatedJwtTokensCache);
-        when(cacheManager.getCache("invalidatedJwtTokens")).thenReturn(invalidatedJwtTokensCache);
+        lenient().when(cacheManager.getCache("validatedJwtTokens")).thenReturn(validatedJwtTokensCache);
+        lenient().when(cacheManager.getCache("invalidatedJwtTokens")).thenReturn(invalidatedJwtTokensCache);
 
         authService = new AuthenticationService(
             applicationContext, authConfigurationProperties, jwtSecurityInitializer,