diff --git a/model/build.gradle b/model/build.gradle index c3c801bd..3fc19fee 100644 --- a/model/build.gradle +++ b/model/build.gradle @@ -58,6 +58,7 @@ project(':model') { testImplementation project(':test') testImplementation project(':external') + testImplementation(testFixtures(project(":drivers"))) testImplementation "ch.qos.logback:logback-classic:_" diff --git a/model/src/main/java/org/gorpipe/gor/driver/providers/stream/sources/mdr/MdrConfiguration.java b/model/src/main/java/org/gorpipe/gor/driver/providers/stream/sources/mdr/MdrConfiguration.java index 98399535..03f61cf6 100644 --- a/model/src/main/java/org/gorpipe/gor/driver/providers/stream/sources/mdr/MdrConfiguration.java +++ b/model/src/main/java/org/gorpipe/gor/driver/providers/stream/sources/mdr/MdrConfiguration.java @@ -23,19 +23,88 @@ package org.gorpipe.gor.driver.providers.stream.sources.mdr; import org.aeonbits.owner.Config; +import org.gorpipe.base.config.ConfigManager; import org.gorpipe.base.config.annotations.Documentation; import org.gorpipe.base.config.converters.DurationConverter; +import org.gorpipe.exceptions.GorParsingException; +import java.nio.file.Files; +import java.nio.file.Path; import java.time.Duration; +import java.util.HashMap; +import java.util.List; +import java.util.Map; public interface MdrConfiguration extends Config { + /** + * Parse MDR credentials from a string. + * + * The credentials are in the format: + * #name\tMdrUrl\tKeycloakUrl\tKeycloakClientId\tKeycloakClientSecret + * \t\t\t\t + * + * # Lines starting with '#' are treated as comments and ignored. + * + * @param credentialsData The credential data + * @return An MdrConfiguration list containing the parsed credentials. + * @throws IllegalArgumentException if the credential string is not in the expected format. + */ + static List parseConfigurationData(String credentialsData) { + List mdrConfList = new java.util.ArrayList<>(); + for (String credLine : credentialsData.split("\n")) { + credLine = credLine.trim(); + if (credLine.isEmpty() || credLine.startsWith("#")) { + continue; + } + + String[] parts = credLine.split("\t"); + if (parts.length != 5) { + throw new IllegalArgumentException("Invalid credential line format. Expected format: \\t\\t\\t"); + } + + mdrConfList.add(ConfigManager.createConfig(MdrConfiguration.class, Map.of( + "GOR_MDR_SERVER_NAME", parts[0], + "GOR_MDR_SERVER", parts[1], + "GOR_KEYCLOAK_SERVER", parts[2], + "GOR_KEYCLOAK_CLIENT_ID", parts[3], + "GOR_KEYCLOAK_CLIENT_SECRET", parts[4] + ))); + } + + return mdrConfList; + } + + static HashMap loadMdrConfigurations(MdrConfiguration defaultConfig) { + HashMap mdrConfigurationsMap = new HashMap<>(); + mdrConfigurationsMap.put(defaultConfig.mdrServerName(), defaultConfig); + + final String MDR_CREDENTIALS_PATH = System.getProperty("gor.mdr.credentials"); + + if (MDR_CREDENTIALS_PATH != null && !MDR_CREDENTIALS_PATH.isEmpty()) { + try { + String credentialsData = Files.readString(Path.of(MDR_CREDENTIALS_PATH)); + for (MdrConfiguration config : parseConfigurationData(credentialsData)) { + mdrConfigurationsMap.put(config.mdrServerName(), config); + } + } catch (Exception e) { + throw new GorParsingException("Failed to read MDR credentials from path: " + MDR_CREDENTIALS_PATH, e); + } + } + return mdrConfigurationsMap; + } + + @Documentation("Name/alias of the MDR server") + @Key("GOR_MDR_SERVER_NAME") + @DefaultValue("default") + String mdrServerName(); + @Documentation("URL to the MDR service") @Key("GOR_MDR_SERVER") - @DefaultValue("https://platform.wuxinextcodedev.com/mdr") + @DefaultValue("https://mdr-service.dev.data.oci.genedx.net") String mdrServer(); - @Documentation("MDR service timout in seconds") + @Documentation("MDR service timeout in seconds") @Key("GOR_MDR_TIMEOUT") @DefaultValue("60") int mdrTimeout(); @@ -57,7 +126,7 @@ public interface MdrConfiguration extends Config { @Documentation("Keycloak auth server url") @Key("GOR_KEYCLOAK_SERVER") - @DefaultValue("https://platform.wuxinextcodedev.com/auth/realms/wuxinextcode.com/protocol/openid-connect/token") + @DefaultValue("https://auth.dev.engops.genedx.net/realms/genedx-dev/protocol/openid-connect/token") String keycloakAuthServer(); @Documentation("Keycloak auth server timout in seconds") @@ -67,7 +136,7 @@ public interface MdrConfiguration extends Config { @Documentation("Keycloak client id") @Key("GOR_KEYCLOAK_CLIENT_ID") - @DefaultValue("gor") + @DefaultValue("dp-gor") String keycloakClientId(); @Documentation("Keycloak service password") diff --git a/model/src/main/java/org/gorpipe/gor/driver/providers/stream/sources/mdr/MdrServer.java b/model/src/main/java/org/gorpipe/gor/driver/providers/stream/sources/mdr/MdrServer.java new file mode 100644 index 00000000..307c6ee3 --- /dev/null +++ b/model/src/main/java/org/gorpipe/gor/driver/providers/stream/sources/mdr/MdrServer.java @@ -0,0 +1,223 @@ +package org.gorpipe.gor.driver.providers.stream.sources.mdr; + +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import org.gorpipe.base.config.ConfigManager; +import org.gorpipe.exceptions.GorResourceException; +import org.gorpipe.exceptions.GorSystemException; +import org.gorpipe.gor.model.SourceRef; +import org.gorpipe.util.http.keycloak.KeycloakClientAuthRequester; +import org.gorpipe.util.http.utils.HttpUtils; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.time.Duration; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.concurrent.TimeUnit; + +public class MdrServer { + + public static final String DEFAULT_MDR_SERVER_NAME = "default"; + + private static final String MDR_PATH = "/api/v1/documents/urls"; + private static final String URL_TYPE = "url_type"; + private static final String URL_TYPE_DIRECT = "direct"; + private static final String URL_TYPE_PRESIGNED = "presigned"; + private static final String INCLUDE_GROUPED = "include_grouped"; + private static final String MDR_ENV = "env"; + + private static final MdrConfiguration defaultConfig = ConfigManager.createPrefixConfig("gor.mdr", MdrConfiguration.class); + + private static HashMap mdrServers; + + private static final Cache documentCache = + CacheBuilder.newBuilder().concurrencyLevel(4).expireAfterAccess(defaultConfig.mdrCacheDuration(), TimeUnit.MINUTES).build(); + + static { + loadMdrServers(defaultConfig); + } + + public static void loadMdrServers(MdrConfiguration defaultConfig) { + mdrServers = MdrConfiguration.loadMdrConfigurations(defaultConfig).entrySet().stream() + .collect(HashMap::new, + (map, entry) -> map.put(entry.getKey(), new MdrServer(entry.getValue())), + HashMap::putAll); + } + + public static String resolveUrl(String url) { + URI uri = URI.create(url); + return mdrServers.get(extractMdrEnvName(uri)).resolveMdrUrl(uri); + } + + public static void cacheUrls(List sources) { + HashMap> sourcesByMdrServer = new HashMap<>(); + for (SourceRef source : sources) { + var mdrServerName = extractMdrEnvName(URI.create(source.file)); + sourcesByMdrServer.computeIfAbsent(mdrServerName, k -> new java.util.ArrayList<>()).add(source); + } + for (var entry : sourcesByMdrServer.keySet()) { + mdrServers.get(entry).cacheMdrUrls(sourcesByMdrServer.get(entry)); + } + } + + private MdrConfiguration config; + private MdrAuthorizedClient authorizedClient; + + public MdrServer(MdrConfiguration config) { + this.config = config; + } + + public MdrAuthorizedClient getAuthorizedClient() { + if (this.authorizedClient == null) { + this.authorizedClient = new MdrAuthorizedClient( + new KeycloakClientAuthRequester(config.keycloakAuthServer(), + Duration.ofSeconds(defaultConfig.keycloakAuthTimeout()), + config.keycloakClientId(), config.keycloakClientSecret()), + Duration.ofSeconds(defaultConfig.mdrTimeout())); + } + return this.authorizedClient; + } + + public String constructPayload(URI mdrUrl) { + return "{\"document_ids\": [\"%s\"]}".formatted(extractDocumentId(mdrUrl)); + } + + public String constructPayload(List sources) { + var sb = new StringBuilder(); + sources.forEach(s -> { + try { + if (MdrSourceType.MDR.match(s.file)) { + var url = new URI(s.file); + sb.append("\"").append(extractDocumentId(url)).append("\","); + } + } catch (URISyntaxException e) { + throw new GorResourceException("Invalid uri: " + s.file, s.file, e); + } + }); + + if (sb.isEmpty()) { + return null; + } + + sb.deleteCharAt(sb.length() - 1); + sb.append("]}"); + return "{\"document_ids\": [" + sb; + } + + public URI constructUrl(URI mdrUrl) throws IOException, URISyntaxException { + var mdrQueryMap = new LinkedHashMap(); + + if (mdrUrl != null) { + var mdrQueryString = mdrUrl.getQuery(); + + if (mdrQueryString != null) { + mdrQueryMap.putAll(HttpUtils.parseQuery(mdrQueryString)); + } + } + + if (!mdrQueryMap.containsKey(URL_TYPE)) { + mdrQueryMap.put(URL_TYPE, defaultConfig.mdrDefaultLinkType()); + } + + if (!mdrQueryMap.containsKey(INCLUDE_GROUPED)) { + mdrQueryMap.put(INCLUDE_GROUPED, defaultConfig.mdrIncludeGrouped() ? "true" : "false"); + } + + var baseUri = URI.create(config.mdrServer()); + + return new URI( baseUri.getScheme(), baseUri.getHost(), baseUri.getPath() + + MDR_PATH, HttpUtils.constructQuery(mdrQueryMap), null); + } + + public String resolveMdrUrl(URI uri) { + var query = uri.getQuery() != null ? uri.getQuery() : ""; + + var mdrDocument = documentCache.getIfPresent(new MdrDocumentCacheKey(extractDocumentId(uri), + query.contains(URL_TYPE_PRESIGNED) ? URL_TYPE_PRESIGNED : URL_TYPE_DIRECT)); + + if (mdrDocument == null) { + var mdrResult = getMdrDocument(uri); + documentCache.put(new MdrDocumentCacheKey(extractDocumentId(uri), mdrResult.url_type()), mdrResult.urls().get(0)); + mdrDocument = mdrResult.urls().get(0); + } + + return mdrDocument.url(); + } + + public void cacheMdrUrls(List sources) { + // Create a list of all sources + var payload = constructPayload(sources); + + if (payload == null) { + return; + } + + // call the mdr service and get the bulk urls + try { + + var mdrUri = constructUrl(null); + var result = getAuthorizedClient().post(mdrUri, payload); + var mdrResult = MdrUrlsResult.fromJSON(result); + + if (mdrResult == null) { + throw new GorResourceException("Invalid response from MDR", mdrUri.toString()); + } + + // Cache all entries + mdrResult.urls().forEach(u -> { + documentCache.put(new MdrDocumentCacheKey(u.document_id(), mdrResult.url_type()), u); + for (var s : sources) { + if (s.file.startsWith("mdr://" + u.document_id())) { + s.file = u.url(); + } + } + }); + } catch (Throwable e) { + // Ignore errors and leave mdr:// urls as is, i.e. not cached. + } + } + + private MdrUrlsResult getMdrDocument(URI url) { + try { + var mdrUri = constructUrl(url); + var payload = constructPayload(url); + + var result = getAuthorizedClient().post(mdrUri, payload); + + var mdrResult = MdrUrlsResult.fromJSON(result); + + if (mdrResult == null) { + throw new GorResourceException("Invalid response from MDR: " + result, url.toString()); + } else if (mdrResult.urls().size() != 1) { + throw new GorResourceException("Invalid response from MDR, only one source allowed per request, got " + mdrResult.urls().size(), url.toString()); + } + return mdrResult; + } catch (URISyntaxException e) { + throw new GorResourceException("Invalid uri: " + url, url.toString(), e); + } catch (IOException e) { + throw new GorResourceException("Error connecting to MDR: " + url, url.toString(), e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new GorSystemException("MDR call interrupted: " + url, e); + } + } + + public static String extractDocumentId(URI mdrUrl) { + return mdrUrl.getHost(); + } + + public static String extractMdrEnvName(URI mdrUrl) { + if (mdrUrl != null) { + var queryString = mdrUrl.getQuery(); + if (queryString != null) { + return HttpUtils.parseQuery(queryString).getOrDefault(MDR_ENV, DEFAULT_MDR_SERVER_NAME); + } + } + + return DEFAULT_MDR_SERVER_NAME; + } + +} diff --git a/model/src/main/java/org/gorpipe/gor/driver/providers/stream/sources/mdr/MdrSourceProvider.java b/model/src/main/java/org/gorpipe/gor/driver/providers/stream/sources/mdr/MdrSourceProvider.java index 72d3f364..840f18be 100644 --- a/model/src/main/java/org/gorpipe/gor/driver/providers/stream/sources/mdr/MdrSourceProvider.java +++ b/model/src/main/java/org/gorpipe/gor/driver/providers/stream/sources/mdr/MdrSourceProvider.java @@ -2,15 +2,9 @@ package org.gorpipe.gor.driver.providers.stream.sources.mdr; import com.google.auto.service.AutoService; -import com.google.common.cache.Cache; -import com.google.common.cache.CacheBuilder; import org.gorpipe.gor.driver.utils.RetryHandlerBase; -import org.gorpipe.util.http.utils.HttpUtils; -import org.gorpipe.util.http.client.AuthorizedHttpClient; -import org.gorpipe.util.http.keycloak.KeycloakClientAuthRequester; import org.gorpipe.base.config.ConfigManager; import org.gorpipe.exceptions.GorResourceException; -import org.gorpipe.exceptions.GorSystemException; import org.gorpipe.gor.driver.SourceProvider; import org.gorpipe.gor.driver.meta.SourceReference; import org.gorpipe.gor.driver.meta.SourceType; @@ -20,36 +14,36 @@ import org.gorpipe.gor.session.GorSession; import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.time.Duration; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.concurrent.TimeUnit; import java.util.stream.Stream; +/** + * MDR Source Provider + * + * Uses the MDR service to resolve document ids to presigned or direct urls. + * + * The MDR url is formatted as: + * + * mdr://[/][?url_type={direct|presigned}&include_grouped={true|false}&env=] + * + * = The MDR document id (UUID format). + * = Optional filename to use when the document contains multiple files. + * + * Query parameters: + * url_type = Optional url type to request. Default is direct + * include_grouped = Optional flag to include grouped files when requesting presigned urls. Default is false. + * env = Optional MDR environment name/alias to use when resolving the document. The env parameter must match the + * configured MDR environments. Default is "default". + * + */ @AutoService(SourceProvider.class) public class MdrSourceProvider extends StreamSourceProvider { - private static final String MDR_PATH = "/api/v1/documents/urls"; - private static final String URL_TYPE = "url_type"; - private static final String URL_TYPE_DIRECT = "direct"; - private static final String URL_TYPE_PRESIGNED = "presigned"; - private static final String INCLUDE_GROUPED = "include_grouped"; - private static final MdrConfiguration config = ConfigManager.createPrefixConfig("gor.mdr", MdrConfiguration.class); - private static final Cache documentCache = - CacheBuilder.newBuilder().concurrencyLevel(4).expireAfterAccess(config.mdrCacheDuration(), TimeUnit.MINUTES).build(); - - private final AuthorizedHttpClient authorizedClient; public MdrSourceProvider() { - this.authorizedClient = new MdrAuthorizedClient( - new KeycloakClientAuthRequester(config.keycloakAuthServer(), - Duration.ofSeconds(config.keycloakAuthTimeout()), - config.keycloakClientId(), config.keycloakClientSecret()), - Duration.ofSeconds(config.mdrTimeout())); + } + @Override public SourceType[] getSupportedSourceTypes() { return new SourceType[]{MdrSourceType.MDR}; @@ -57,51 +51,15 @@ public SourceType[] getSupportedSourceTypes() { @Override public StreamSource resolveDataSource(SourceReference sourceReference) throws IOException { - try { - URI url = new URI(sourceReference.getUrl()); - - var query = url.getQuery(); - - if (query == null) - query = ""; - - var cached = - documentCache.getIfPresent(new MdrDocumentCacheKey(url.getHost(), query.contains(URL_TYPE_PRESIGNED) ? URL_TYPE_PRESIGNED : URL_TYPE_DIRECT)); - - var session = GorSession.currentSession.get(); - - if (session == null) { - throw new GorResourceException("No session found", sourceReference.getUrl()); - } - - if (cached != null) { - return (StreamSource)session.getProjectContext().getFileReader().resolveUrl(cached.url()); - } - - var mdrUri = constructUrl(url); - var payload = constructPayload(url); + var session = GorSession.currentSession.get(); - var result = this.authorizedClient.post(mdrUri, payload); - - var mdrResult = MdrUrlsResult.fromJSON(result); - - if (mdrResult == null) { - throw new GorResourceException("Invalid response from MDR: " + result, sourceReference.getUrl()); - } else if (mdrResult.urls().size() != 1) { - throw new GorResourceException("Invalid response from MDR, only one source allowed per requerst, got " + mdrResult.urls().size(), sourceReference.getUrl()); - } + if (session == null) { + throw new GorResourceException("No session found", sourceReference.getUrl()); + } - documentCache.put(new MdrDocumentCacheKey(url.getHost(), mdrResult.url_type()), mdrResult.urls().get(0)); + String documentUrl = MdrServer.resolveUrl(sourceReference.getUrl()); - return (StreamSource)session.getProjectContext().getFileReader().resolveUrl(mdrResult.urls().get(0).url()); - } catch (URISyntaxException e) { - throw new GorResourceException("Invalid uri: " + sourceReference.getUrl(), sourceReference.getUrl(), e); - } catch (IOException e) { - throw new GorResourceException("Error connecting to MDR: " + sourceReference.getUrl(), sourceReference.getUrl(), e); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new GorSystemException("MDR call interrupted: " + sourceReference.getUrl(), e); - } + return (StreamSource)session.getProjectContext().getFileReader().resolveUrl(documentUrl); } @Override @@ -114,93 +72,11 @@ protected RetryHandlerBase getRetryHandler() { return retryHandler; } - private String constructPayload(URI mdrUrl) { - return "{\"document_ids\": [\"" + mdrUrl.getHost() + "\"]}"; - } - - private String constructPayload(List sources) { - var sb = new StringBuilder(); - sources.forEach(s -> { - try { - if (MdrSourceType.MDR.match(s.file)) { - var url = new URI(s.file); - sb.append("\"").append(url.getHost()).append("\","); - } - } catch (URISyntaxException e) { - throw new GorResourceException("Invalid uri: " + s.file, s.file, e); - } - }); - - if (sb.isEmpty()) { - return null; - } - - sb.deleteCharAt(sb.length() - 1); - sb.append("]}"); - return "{\"document_ids\": [" + sb; - } - - private URI constructUrl(URI mdrUrl) throws IOException, URISyntaxException { - var mdrQueryMap = new LinkedHashMap(); - - if (mdrUrl != null) { - var mdrQueryString = mdrUrl.getQuery(); - - if (mdrQueryString != null) { - mdrQueryMap.putAll(HttpUtils.parseQuery(mdrQueryString)); - } - } - - if (!mdrQueryMap.containsKey(URL_TYPE)) { - mdrQueryMap.put(URL_TYPE, config.mdrDefaultLinkType()); - } - - if (!mdrQueryMap.containsKey(INCLUDE_GROUPED)) { - mdrQueryMap.put(INCLUDE_GROUPED, config.mdrIncludeGrouped() ? "true" : "false"); - } - - var baseUri = URI.create(config.mdrServer()); - - return new URI( baseUri.getScheme(), baseUri.getHost(), baseUri.getPath() + - MDR_PATH, HttpUtils.constructQuery(mdrQueryMap), null); - } - @Override public Stream prepareSources(Stream sources) { - var sourcesList = sources.toList(); - - // Create a list of all sources - var payload = constructPayload(sourcesList); - - if (payload == null) { - return sourcesList.stream(); - } - - // call the mdr service and get the bulk urls - try { - var mdrUri = constructUrl(null); - var result = this.authorizedClient.post(mdrUri, payload); - var mdrResult = MdrUrlsResult.fromJSON(result); - - if (mdrResult == null) { - throw new GorResourceException("Invalid response from MDR", mdrUri.toString()); - } - - // Cache all entries - mdrResult.urls().forEach(u -> { - documentCache.put(new MdrDocumentCacheKey(u.document_id(), mdrResult.url_type()), u); - for (var s : sourcesList) { - if (s.file.startsWith("mdr://" + u.document_id())) { - s.file = u.url(); - } - } - }); - - return sourcesList.stream(); - } catch (Throwable e) { - return sourcesList.stream(); - } + MdrServer.cacheUrls(sourcesList); + return sourcesList.stream(); } } diff --git a/model/src/main/java/org/gorpipe/gor/model/DriverBackedFileReader.java b/model/src/main/java/org/gorpipe/gor/model/DriverBackedFileReader.java index 888e4b50..1c51c415 100644 --- a/model/src/main/java/org/gorpipe/gor/model/DriverBackedFileReader.java +++ b/model/src/main/java/org/gorpipe/gor/model/DriverBackedFileReader.java @@ -35,6 +35,7 @@ import org.gorpipe.gor.driver.meta.DataType; import org.gorpipe.gor.driver.meta.SourceReference; import org.gorpipe.gor.driver.meta.SourceReferenceBuilder; +import org.gorpipe.gor.driver.meta.SourceType; import org.gorpipe.gor.driver.providers.rows.RowIteratorSource; import org.gorpipe.gor.driver.providers.stream.StreamSourceFile; import org.gorpipe.gor.driver.providers.stream.StreamSourceProvider; @@ -50,6 +51,7 @@ import org.slf4j.LoggerFactory; import java.io.*; +import java.net.URI; import java.nio.file.*; import java.nio.file.attribute.FileAttribute; import java.util.*; @@ -460,20 +462,25 @@ public Stream prepareSources(Stream sources) { } var gorDriverFactory = (PluggableGorDriver)GorDriverFactory.fromConfig(); - var sourceTypes = gorDriverFactory.getSupportedSourceTypes(); - var resultStream = sources; - - for (var sourceType : sourceTypes) { - if (!sourceType.supportsPreparation()) { - continue; + HashMap> sourcesBySourceType = new HashMap<>(); + sources.forEach(source -> { + try (var dataSource = resolveUrl(source.file)) { + sourcesBySourceType.computeIfAbsent(dataSource.getSourceType(), k -> new java.util.ArrayList<>()).add(source); } + }); - var provider =gorDriverFactory.getSourceProvider(sourceType); - resultStream = provider.prepareSources(resultStream); + var outSources = new ArrayList(); + for (var sourceType : sourcesBySourceType.keySet()) { + if (sourceType.supportsPreparation()) { + var provider = gorDriverFactory.getSourceProvider(sourceType); + outSources.addAll(provider.prepareSources(sourcesBySourceType.get(sourceType).stream()).toList()); + } else { + outSources.addAll(sourcesBySourceType.get(sourceType)); + } } - return resultStream; + return outSources.stream(); } private Stream getDependentDestStream(String source, String dest, DataSource sourceSource) throws IOException { diff --git a/model/src/test/java/org/gorpipe/gor/driver/providers/mdr/UTestMDR.java b/model/src/test/java/org/gorpipe/gor/driver/providers/mdr/UTestMDR.java index eb809a09..0b821212 100644 --- a/model/src/test/java/org/gorpipe/gor/driver/providers/mdr/UTestMDR.java +++ b/model/src/test/java/org/gorpipe/gor/driver/providers/mdr/UTestMDR.java @@ -1,13 +1,20 @@ package org.gorpipe.gor.driver.providers.mdr; import gorsat.TestUtils; +import org.gorpipe.base.config.ConfigManager; +import org.gorpipe.base.security.Credentials; import org.gorpipe.exceptions.GorResourceException; +import org.gorpipe.gor.driver.providers.stream.sources.mdr.MdrConfiguration; +import org.gorpipe.gor.driver.providers.stream.sources.mdr.MdrServer; import org.gorpipe.test.IntegrationTests; +import org.gorpipe.utils.DriverUtils; import org.junit.*; import org.junit.contrib.java.lang.system.EnvironmentVariables; +import org.junit.contrib.java.lang.system.RestoreSystemProperties; import org.junit.experimental.categories.Category; import java.io.*; +import java.net.URI; import java.nio.file.Files; import java.nio.file.Path; import java.util.Properties; @@ -16,113 +23,120 @@ @Ignore("Can not access Keycloak from the the Gitlab build servers.") public class UTestMDR { - private static String S3_KEY; - private static String S3_SECRET; - private static String S3_REGION = "eu-west-1"; + private static String S3_REGION = "us-ashburn-1"; + private static final String S3_BUCKET = "mdr-genomic-data-dev"; + private static MdrConfiguration config; + private static String securityContext; @Rule public final EnvironmentVariables environmentVariables = new EnvironmentVariables(); + @Rule + public final RestoreSystemProperties restoreSystemProperties = new RestoreSystemProperties(); + + @BeforeClass - public static void setupClass() { + public static void setupClass() throws IOException { Properties props = TestUtils.loadSecrets(); var secret = props.getProperty("GOR_KEYCLOAK_CLIENT_SECRET"); if (secret != null) { System.setProperty("GOR_KEYCLOAK_CLIENT_SECRET", secret); } - S3_KEY = props.getProperty("S3_KEY"); - S3_SECRET = props.getProperty("S3_SECRET"); - + securityContext = DriverUtils.createSecurityContext( + "s3", + S3_BUCKET, + Credentials.OwnerType.System, "", + props.getProperty("MDR_S3_KEY"), + props.getProperty("MDR_S3_SECRET"), + props.getProperty("MDR_S3_ENDPOINT"), + ""); + + config = ConfigManager.createPrefixConfig("gor.mdr", MdrConfiguration.class); + + Path credFile = Files.createTempFile("gor.mdr.credentials", ".tmp"); + credFile.toFile().deleteOnExit(); + Files.writeString(credFile, """ + #name\tMdrUrl\tKeycloakUrl\tKeycloakClientId\tKeycloakClientSecret + dev\t%s\t%s\t%s\t%s + """.formatted(config.mdrServer(), config.keycloakAuthServer(), config.keycloakClientId(), secret)); + System.setProperty("gor.mdr.credentials", credFile.toString()); + MdrServer.loadMdrServers(config); } @Test - public void testReadDocument() { - - environmentVariables.set("AWS_ACCESS_KEY_ID", S3_KEY); - environmentVariables.set("AWS_SECRET_ACCESS_KEY", S3_SECRET); - environmentVariables.set("AWS_REGION", S3_REGION); - - var result = TestUtils.runGorPipeLines("gor mdr://2806e2ec-30f0-41f1-bdf1-ce3b2def078a | top 10000"); - - Assert.assertEquals(10001, result.length); - Assert.assertEquals("Chrom\tgene_start\tgene_end\tGene_Symbol\n", result[0]); - Assert.assertEquals("chr1\t11868\t14412\tDDX11L1\n", result[1]); - Assert.assertEquals("chr1\t35899090\t36023551\tKIAA0319L\n", result[1009]); - + public void testStdUrl() { + Assert.assertEquals("default", MdrServer.extractMdrEnvName(URI.create("mdr://ff8e31e0-a9ae-41eb-bb8d-21854a47d8b3"))); + Assert.assertEquals("ff8e31e0-a9ae-41eb-bb8d-21854a47d8b3", MdrServer.extractDocumentId(URI.create("mdr://ff8e31e0-a9ae-41eb-bb8d-21854a47d8b3"))); + Assert.assertEquals("default", MdrServer.extractMdrEnvName(URI.create("mdr://ff8e31e0-a9ae-41eb-bb8d-21854a47d8b3/test.txt"))); + Assert.assertEquals("ff8e31e0-a9ae-41eb-bb8d-21854a47d8b3", MdrServer.extractDocumentId(URI.create("mdr://ff8e31e0-a9ae-41eb-bb8d-21854a47d8b3/test.txt"))); } @Test - public void testReadDocumentWithNor() { - - environmentVariables.set("AWS_ACCESS_KEY_ID", S3_KEY); - environmentVariables.set("AWS_SECRET_ACCESS_KEY", S3_SECRET); - environmentVariables.set("AWS_REGION", S3_REGION); - - var result = TestUtils.runGorPipeLines("nor -h mdr://2806e2ec-30f0-41f1-bdf1-ce3b2def078a | top 10000"); - - Assert.assertEquals(10001, result.length); - Assert.assertEquals("ChromNOR\tPosNOR\tChrom\tgene_start\tgene_end\tGene_Symbol\n", result[0]); - Assert.assertEquals("chrN\t0\tchr1\t11868\t14412\tDDX11L1\n", result[1]); - Assert.assertEquals("chrN\t0\tchr1\t35899090\t36023551\tKIAA0319L\n", result[1009]); + public void tesServerUrl() { + Assert.assertEquals("dev", MdrServer.extractMdrEnvName(URI.create("mdr://ff8e31e0-a9ae-41eb-bb8d-21854a47d8b3?env=dev"))); + Assert.assertEquals("ff8e31e0-a9ae-41eb-bb8d-21854a47d8b3", MdrServer.extractDocumentId(URI.create("mdr://ff8e31e0-a9ae-41eb-bb8d-21854a47d8b3?env=dev"))); + Assert.assertEquals("dev", MdrServer.extractMdrEnvName(URI.create("mdr://ff8e31e0-a9ae-41eb-bb8d-21854a47d8b3/test.txt?env=dev"))); + Assert.assertEquals("ff8e31e0-a9ae-41eb-bb8d-21854a47d8b3", MdrServer.extractDocumentId(URI.create("mdr://ff8e31e0-a9ae-41eb-bb8d-21854a47d8b3/test.txt?env=dev"))); } @Test - public void testReadDocuments() { + public void testReadDocumentStd() { + var result = TestUtils.runGorPipe("nor mdr://ff8e31e0-a9ae-41eb-bb8d-21854a47d8b3 | top 1", true, securityContext).split("\n"); - environmentVariables.set("AWS_ACCESS_KEY_ID", S3_KEY); - environmentVariables.set("AWS_SECRET_ACCESS_KEY", S3_SECRET); - environmentVariables.set("AWS_REGION", S3_REGION); + Assert.assertEquals(2, result.length); + Assert.assertEquals("ChromNOR\tPosNOR\tcol1", result[0]); + Assert.assertEquals("chrN\t0\tVARIANT CALLER SUMMARY,,Number of samples,1", result[1]); + } - var result = TestUtils.runGorPipeLines("gor mdr://2806e2ec-30f0-41f1-bdf1-ce3b2def078a mdr://191b3d28-4db9-4aa3-aa6b-cbb2968885a5 mdr://ee4d7e36-e6dc-42d8-8f78-714242a8cf6d | top 10000"); + @Test + public void testReadDocumentServer() { + var result = TestUtils.runGorPipe("nor mdr://ff8e31e0-a9ae-41eb-bb8d-21854a47d8b3?env=dev | top 1", true, securityContext).split("\n"); - Assert.assertEquals(10001, result.length); - Assert.assertEquals(10001, result.length); - Assert.assertEquals("Chrom\tgene_start\tgene_end\tGene_Symbol\n", result[0]); - Assert.assertEquals("chr1\t11868\t14412\tDDX11L1\n", result[1]); - Assert.assertEquals("chr1\t35519068\t35524872\tRP11-248I9.2\n", result[3000]); + Assert.assertEquals(2, result.length); + Assert.assertEquals("ChromNOR\tPosNOR\tcol1", result[0]); + Assert.assertEquals("chrN\t0\tVARIANT CALLER SUMMARY,,Number of samples,1", result[1]); } @Test public void testReadDocumentThroughLinkFile() throws IOException { - environmentVariables.set("AWS_ACCESS_KEY_ID", S3_KEY); - environmentVariables.set("AWS_SECRET_ACCESS_KEY", S3_SECRET); - environmentVariables.set("AWS_REGION", S3_REGION); - - Path tempFile = Files.createTempFile("document_", ".gor.link"); - Files.write(tempFile, "mdr://2806e2ec-30f0-41f1-bdf1-ce3b2def078a".getBytes()); + Path tempFile = Files.createTempFile("document_", ".nor.link"); + Files.write(tempFile, "mdr://ff8e31e0-a9ae-41eb-bb8d-21854a47d8b3".getBytes()); - var result = TestUtils.runGorPipeLines("gor " + tempFile + " | top 10000"); + var result = TestUtils.runGorPipe("nor " + tempFile + " | top 1", true, securityContext).split("\n"); - Assert.assertEquals(10001, result.length); - Assert.assertEquals("Chrom\tgene_start\tgene_end\tGene_Symbol\n", result[0]); - Assert.assertEquals("chr1\t11868\t14412\tDDX11L1\n", result[1]); - Assert.assertEquals("chr1\t35899090\t36023551\tKIAA0319L\n", result[1009]); + Assert.assertEquals(2, result.length); + Assert.assertEquals("ChromNOR\tPosNOR\tcol1", result[0]); + Assert.assertEquals("chrN\t0\tVARIANT CALLER SUMMARY,,Number of samples,1", result[1]); } @Test public void testNonExistingDocumentId() { - - environmentVariables.set("AWS_ACCESS_KEY_ID", S3_KEY); - environmentVariables.set("AWS_SECRET_ACCESS_KEY", S3_SECRET); - environmentVariables.set("AWS_REGION", S3_REGION); - var ex = Assert.assertThrows(GorResourceException.class, () -> { - TestUtils.runGorPipeLines("gor mdr://f3658d3a-5220-4094-b7b1-311804df3db8"); + TestUtils.runGorPipe("gor mdr://f3658d3a-5220-4094-b7b1-311804df3db8", true, securityContext); }); Assert.assertTrue(ex.getMessage().contains("not found")); } @Test - public void testDictionaryWithInvalidEntry() throws IOException { - environmentVariables.set("AWS_ACCESS_KEY_ID", S3_KEY); - environmentVariables.set("AWS_SECRET_ACCESS_KEY", S3_SECRET); - environmentVariables.set("AWS_REGION", S3_REGION); + @Ignore("Need find new test documents.") + public void testReadDocuments() { + var result = TestUtils.runGorPipe("gor mdr://2806e2ec-30f0-41f1-bdf1-ce3b2def078a mdr://191b3d28-4db9-4aa3-aa6b-cbb2968885a5 mdr://ee4d7e36-e6dc-42d8-8f78-714242a8cf6d | top 10000", true, securityContext).split("\n"); + Assert.assertEquals(10001, result.length); + Assert.assertEquals(10001, result.length); + Assert.assertEquals("Chrom\tgene_start\tgene_end\tGene_Symbol\n", result[0]); + Assert.assertEquals("chr1\t11868\t14412\tDDX11L1\n", result[1]); + Assert.assertEquals("chr1\t35519068\t35524872\tRP11-248I9.2\n", result[3000]); + } + + @Test + @Ignore + public void testDictionaryWithInvalidEntry() throws IOException { // Copy test file to a new file Path tempFile = Files.createTempFile("genes_", ".gord"); Files.copy(Path.of("../tests/data/mdr/genes_mdr_1000.gord"), tempFile, java.nio.file.StandardCopyOption.REPLACE_EXISTING); @@ -134,20 +148,16 @@ public void testDictionaryWithInvalidEntry() throws IOException { var ex = Assert.assertThrows(GorResourceException.class, () -> { var query = "gor " + tempFile + " | top 10000"; - TestUtils.runGorPipeLines(query); + TestUtils.runGorPipe(query, true, securityContext); }); Assert.assertTrue(ex.getMessage().contains("not found")); } @Test + @Ignore public void test1000EntryDictionary() { - environmentVariables.set("AWS_ACCESS_KEY_ID", S3_KEY); - environmentVariables.set("AWS_SECRET_ACCESS_KEY", S3_SECRET); - environmentVariables.set("AWS_REGION", S3_REGION); - - var result = TestUtils.runGorPipeLines("gor ../tests/data/mdr/genes_mdr_1000.gord | top 10000"); - + var result = TestUtils.runGorPipe("gor ../tests/data/mdr/genes_mdr_1000.gord | top 10000", true, securityContext).split("\n"); Assert.assertEquals(10001, result.length); } }