diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 41262afd59..41d107c3db 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -997,7 +997,6 @@ jobs: caching-service: image: ghcr.io/balhar-jakub/caching-service:${{ github.run_id }}-${{ github.run_number }} env: - ZWE_CACHING_SERVICE_PERSISTENT: 'infinispan' CACHING_STORAGE_MODE: "infinispan" JGROUPS_BIND_ADDRESS: "caching-service" JGROUPS_BIND_PORT: "7600" @@ -2164,7 +2163,6 @@ jobs: caching-service: image: ghcr.io/balhar-jakub/caching-service:${{ github.run_id }}-${{ github.run_number }} env: - ZWE_CACHING_SERVICE_PERSISTENT: 'infinispan' JGROUPS_BIND_PORT: "7600" SERVER_SSL_KEYSTORETYPE: "PKCS12" CACHING_STORAGE_INFINISPAN_PERSISTENCE_DATALOCATION: "data_replica" @@ -2175,7 +2173,6 @@ jobs: caching-service-2: image: ghcr.io/balhar-jakub/caching-service:${{ github.run_id }}-${{ github.run_number }} env: - ZWE_CACHING_SERVICE_PERSISTENT: 'infinispan' JGROUPS_BIND_PORT: "7600" SERVER_SSL_KEYSTORETYPE: "PKCS12" CACHING_STORAGE_INFINISPAN_PERSISTENCE_DATALOCATION: "data" @@ -2250,13 +2247,13 @@ jobs: APIML_SECURITY_X509_ENABLED: true APIML_SECURITY_SSL_NONSTRICTVERIFYSSLCERTIFICATESOFSERVICES: true APIML_DISCOVERY_ALLPEERSURLS: https://apiml-2:10011/eureka,https://apiml:10011/eureka - ZWE_CACHING_SERVICE_PERSISTENT: 'infinispan' - JGROUPS_BIND_PORT: "7600" SERVER_SSL_KEYSTORETYPE: "PKCS12" - CACHING_STORAGE_INFINISPAN_INITIALHOSTS: "apiml[7600],apiml-2[7600]" + CACHING_STORAGE_INFINISPAN_NUMSEGMENTS: "16" CACHING_STORAGE_MODE: "infinispan" + CACHING_STORAGE_INFINISPAN_INITIALHOSTS: "apiml[7600],apiml-2[7600]" + JGROUPS_BIND_PORT: "7600" JGROUPS_BIND_ADDRESS: "apiml" - CACHING_STORAGE_INFINISPAN_NUMSEGMENTS: "16" + JGROUPS_KEYEXCHANGE_PORT: 7601 apiml-2: image: ghcr.io/balhar-jakub/apiml:${{ github.run_id }}-${{ github.run_number }} env: @@ -2264,15 +2261,15 @@ jobs: APIML_GATEWAY_SERVICESTOLIMITREQUESTRATE: discoverableclient APIML_SECURITY_SSL_NONSTRICTVERIFYSSLCERTIFICATESOFSERVICES: true APIML_DISCOVERY_ALLPEERSURLS: https://apiml:10011/eureka,https://apiml-2:10011/eureka - ZWE_CACHING_SERVICE_PERSISTENT: 'infinispan' - JGROUPS_BIND_PORT: "7600" SERVER_SSL_KEYSTORETYPE: "PKCS12" - CACHING_STORAGE_INFINISPAN_INITIALHOSTS: "apiml[7600],apiml-2[7600]" - CACHING_STORAGE_MODE: "infinispan" - JGROUPS_BIND_ADDRESS: "apiml-2" APIML_SERVICE_HOSTNAME: "apiml-2" logbackService: ZWEAGW2 CACHING_STORAGE_INFINISPAN_NUMSEGMENTS: "16" + CACHING_STORAGE_MODE: "infinispan" + CACHING_STORAGE_INFINISPAN_INITIALHOSTS: "apiml[7600],apiml-2[7600]" + JGROUPS_BIND_PORT: "7600" + JGROUPS_BIND_ADDRESS: "apiml-2" + JGROUPS_KEYEXCHANGE_PORT: 7601 api-catalog-services: image: ghcr.io/balhar-jakub/api-catalog-services:${{ github.run_id }}-${{ github.run_number }} volumes: @@ -2296,7 +2293,8 @@ jobs: - name: Run Startup Check if: always() run: > - ./gradlew runStartUpCheck --info -Denvironment.config=-docker-modulith + ./gradlew runStartUpCheck --info -Denvironment.config=-docker-modulith-ha + -Dgateway.instances=2 -Ddiscovery.instances=2 -Dcaching.instances=2 -Dapicatalog.instances=2 -Denvironment.modulith=true -Ddiscoverableclient.instances=1 -DcentralGateway.instances=0 -Partifactory_user=$ARTIFACTORY_USERNAME -Partifactory_password=$ARTIFACTORY_PASSWORD env: diff --git a/apiml/src/main/java/org/zowe/apiml/ApimlApplication.java b/apiml/src/main/java/org/zowe/apiml/ApimlApplication.java index 174d94a367..cc9212973b 100644 --- a/apiml/src/main/java/org/zowe/apiml/ApimlApplication.java +++ b/apiml/src/main/java/org/zowe/apiml/ApimlApplication.java @@ -12,6 +12,7 @@ import org.springframework.boot.SpringApplication; import org.springframework.boot.actuate.autoconfigure.logging.OpenTelemetryLoggingAutoConfiguration; +import org.springframework.boot.actuate.autoconfigure.metrics.cache.CacheMetricsAutoConfiguration; import org.springframework.boot.actuate.autoconfigure.opentelemetry.OpenTelemetryAutoConfiguration; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.autoconfigure.security.oauth2.client.reactive.ReactiveOAuth2ClientAutoConfiguration; @@ -28,7 +29,8 @@ ReactiveOAuth2ClientAutoConfiguration.class, OpenTelemetryAutoConfiguration.class, OpenTelemetryLoggingAutoConfiguration.class, - io.opentelemetry.instrumentation.spring.autoconfigure.OpenTelemetryAutoConfiguration.class + io.opentelemetry.instrumentation.spring.autoconfigure.OpenTelemetryAutoConfiguration.class, + CacheMetricsAutoConfiguration.class }, scanBasePackages = { "org.zowe.apiml.filter", diff --git a/caching-service/src/main/java/org/zowe/apiml/caching/service/infinispan/config/InfinispanConfig.java b/caching-service/src/main/java/org/zowe/apiml/caching/service/infinispan/config/InfinispanConfig.java index f82908a5a0..0f66feb596 100644 --- a/caching-service/src/main/java/org/zowe/apiml/caching/service/infinispan/config/InfinispanConfig.java +++ b/caching-service/src/main/java/org/zowe/apiml/caching/service/infinispan/config/InfinispanConfig.java @@ -43,11 +43,9 @@ import java.io.InputStream; import java.nio.file.Paths; import java.time.Duration; -import java.util.*; +import java.util.HashMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; -import java.util.stream.Collectors; -import java.util.stream.Stream; import static org.zowe.apiml.security.SecurityUtils.formatKeyringUrl; import static org.zowe.apiml.security.SecurityUtils.isKeyring; @@ -211,12 +209,11 @@ synchronized LazyCacheManager cacheManager(ResourceLoader resourceLoader, Applic System.setProperty("infinispan.ssl.trustStore", trustStore); System.setProperty("infinispan.ssl.trustStorePassword", trustStorePass); - var caches = Stream.of(CACHE_ZOWE, CACHE_ZOWE_INVALIDATED_TOKEN) - .collect(Collectors.toMap( cacheName -> cacheName, cacheName -> getDistributedCacheConfig())); + var caches = new HashMap(); + caches.put(CACHE_ZOWE, getDistributedCacheConfig()); + caches.put(CACHE_ZOWE_INVALIDATED_TOKEN, getDistributedCacheConfig()); if (applicationInfo.isModulith()) { - caches.put(CACHE_ZOWE, getDistributedCacheConfig()); - caches.put(CACHE_ZOWE_INVALIDATED_TOKEN, getDistributedCacheConfig()); caches.put("invalidatedJwtTokens", getDistributedCacheConfig()); // 1 minute to force zosmf tokens validation against zosmf for invalidated tokens diff --git a/zaas-service/src/main/java/org/zowe/apiml/zaas/security/service/AuthenticationService.java b/zaas-service/src/main/java/org/zowe/apiml/zaas/security/service/AuthenticationService.java index 99a3a2d317..41155c63b7 100644 --- a/zaas-service/src/main/java/org/zowe/apiml/zaas/security/service/AuthenticationService.java +++ b/zaas-service/src/main/java/org/zowe/apiml/zaas/security/service/AuthenticationService.java @@ -51,12 +51,8 @@ import org.zowe.apiml.zaas.security.service.zosmf.ZosmfService; import java.text.ParseException; -import java.util.Arrays; -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; -import java.util.Set; +import java.util.*; +import java.util.concurrent.atomic.AtomicReference; import static org.zowe.apiml.security.common.util.JwtUtils.getJwtClaims; import static org.zowe.apiml.security.common.util.JwtUtils.handleJwtParserException; @@ -89,14 +85,38 @@ public class AuthenticationService { private final CacheManager cacheManager; private final CacheUtils cacheUtils; private boolean isModulithMode; - private Cache validatedJwtTokensCache; - private Cache invalidatedJwtTokensCache; + private final AtomicReference validatedJwtTokensCache = new AtomicReference<>(); + private final AtomicReference invalidatedJwtTokensCache = new AtomicReference<>(); @PostConstruct public void afterPropertiesSet() { isModulithMode = applicationContext.containsBean("modulithConfig"); - validatedJwtTokensCache = cacheManager.getCache(CACHE_VALIDATED_JWT_TOKENS); - invalidatedJwtTokensCache = cacheManager.getCache(CACHE_INVALIDATED_JWT_TOKENS); + } + + private Cache getValidatedJwtTokensCache() { + var cacheValidatedJwtTokensCache = this.validatedJwtTokensCache.get(); + if (cacheValidatedJwtTokensCache == null) { + synchronized (validatedJwtTokensCache) { + cacheValidatedJwtTokensCache = validatedJwtTokensCache.get(); + if (cacheValidatedJwtTokensCache == null) { + validatedJwtTokensCache.set(cacheManager.getCache(CACHE_VALIDATED_JWT_TOKENS)); + } + } + } + return validatedJwtTokensCache.get(); + } + + private Cache getInvalidatedJwtTokensCache() { + var cacheInvalidatedJwtTokensCache = this.invalidatedJwtTokensCache.get(); + if (cacheInvalidatedJwtTokensCache == null) { + synchronized (invalidatedJwtTokensCache) { + cacheInvalidatedJwtTokensCache = this.invalidatedJwtTokensCache.get(); + if (cacheInvalidatedJwtTokensCache == null) { + invalidatedJwtTokensCache.set(cacheManager.getCache(CACHE_INVALIDATED_JWT_TOKENS)); + } + } + } + return invalidatedJwtTokensCache.get(); } /** @@ -229,20 +249,23 @@ private Boolean doInvalidate(String jwtToken, boolean distribute, Application ap } private void putValidationCache(String jwtToken, TokenAuthentication tokenAuthentication) { - if (jwtToken != null && validatedJwtTokensCache != null) { - validatedJwtTokensCache.put(jwtToken, tokenAuthentication); + var cacheValidatedJwtTokens = getValidatedJwtTokensCache(); + if (jwtToken != null && cacheValidatedJwtTokens != null) { + cacheValidatedJwtTokens.put(jwtToken, tokenAuthentication); } } private void evictValidationCache(String jwtToken) { - if (validatedJwtTokensCache != null) { - validatedJwtTokensCache.evict(jwtToken); + var cacheValidatedJwtTokens = getValidatedJwtTokensCache(); + if (cacheValidatedJwtTokens != null) { + cacheValidatedJwtTokens.evict(jwtToken); } } private void putInvalidatedCache(String jwtToken) { - if (invalidatedJwtTokensCache != null) { - invalidatedJwtTokensCache.put(jwtToken, Boolean.TRUE); + var cacheInvalidatedJwtTokens = getInvalidatedJwtTokensCache(); + if (cacheInvalidatedJwtTokens != null) { + cacheInvalidatedJwtTokens.put(jwtToken, Boolean.TRUE); } } @@ -307,10 +330,11 @@ private boolean invalidateTokenOnAnotherInstance(String jwtToken, Application ap * @return true - token is invalidated, otherwise token is still valid */ public boolean isInvalidated(String jwtToken) { - if (invalidatedJwtTokensCache == null) { + var cacheInvalidatedJwtTokens = getInvalidatedJwtTokensCache(); + if (cacheInvalidatedJwtTokens == null) { return false; } - Cache.ValueWrapper wrapper = invalidatedJwtTokensCache.get(jwtToken); + Cache.ValueWrapper wrapper = cacheInvalidatedJwtTokens.get(jwtToken); boolean result = wrapper != null && Boolean.TRUE.equals(wrapper.get()); log.debug("Token invalidation check for ...{}: {}", StringUtils.right(jwtToken, 15), result); return result; @@ -361,8 +385,9 @@ public TokenAuthentication validateJwtToken(String jwtToken) { throw new TokenNotValidException("Token ...%s was invalidated.".formatted(StringUtils.right(jwtToken, 15))); } - if (validatedJwtTokensCache != null) { - Cache.ValueWrapper cached = validatedJwtTokensCache.get(jwtToken); + var cacheValidatedJwtTokens = getValidatedJwtTokensCache(); + if (cacheValidatedJwtTokens != null) { + Cache.ValueWrapper cached = cacheValidatedJwtTokens.get(jwtToken); if (cached != null) { var tokenAuthentication = (TokenAuthentication) cached.get(); log.debug("JWT ...{} found in the cache. Is authenticated: {}", StringUtils.right(jwtToken, 15), tokenAuthentication.isAuthenticated()); diff --git a/zaas-service/src/test/java/org/zowe/apiml/zaas/security/service/AuthenticationServiceTest.java b/zaas-service/src/test/java/org/zowe/apiml/zaas/security/service/AuthenticationServiceTest.java index 6eeffda6c4..e2fe08c42c 100644 --- a/zaas-service/src/test/java/org/zowe/apiml/zaas/security/service/AuthenticationServiceTest.java +++ b/zaas-service/src/test/java/org/zowe/apiml/zaas/security/service/AuthenticationServiceTest.java @@ -127,8 +127,8 @@ void setup() { authConfigurationProperties = new AuthConfigurationProperties(); authConfigurationProperties.getZosmf().setServiceId(ZOSMF); - when(cacheManager.getCache("validatedJwtTokens")).thenReturn(validatedJwtTokensCache); - when(cacheManager.getCache("invalidatedJwtTokens")).thenReturn(invalidatedJwtTokensCache); + lenient().when(cacheManager.getCache("validatedJwtTokens")).thenReturn(validatedJwtTokensCache); + lenient().when(cacheManager.getCache("invalidatedJwtTokens")).thenReturn(invalidatedJwtTokensCache); authService = new AuthenticationService( applicationContext, authConfigurationProperties, jwtSecurityInitializer,