From b96e1fa1ef92ba952e29ab12db1c2a8368530dae Mon Sep 17 00:00:00 2001 From: Gisli Magnusson Date: Fri, 30 Jan 2026 01:05:19 +0000 Subject: [PATCH 1/2] fix(ENGKNOW-3126): Defaults for MDR config location (sneaked in minor improvments for tee testing). --- .../org/gorpipe/s3/shared/ITestS3Shared.java | 29 +++++++++++++++++++ .../src/test/java/gorsat/UTestGorTee.java | 8 ++--- .../stream/sources/mdr/MdrConfiguration.java | 2 +- .../stream/sources/mdr/MdrServer.java | 19 ++++++++---- 4 files changed, 48 insertions(+), 10 deletions(-) diff --git a/drivers/src/test/java/org/gorpipe/s3/shared/ITestS3Shared.java b/drivers/src/test/java/org/gorpipe/s3/shared/ITestS3Shared.java index 4e8250467..2e1026b38 100644 --- a/drivers/src/test/java/org/gorpipe/s3/shared/ITestS3Shared.java +++ b/drivers/src/test/java/org/gorpipe/s3/shared/ITestS3Shared.java @@ -598,6 +598,35 @@ public void testWriteExplicitWrite() throws IOException { } } + @Test + public void testWriteWithTeeAndLinks() throws IOException { + String securityContext = createSecurityContext("s3data", Credentials.OwnerType.System, "some_env", S3_KEY, S3_SECRET); + String gorRoot = Path.of(workDir.getRoot().toString(), "some_project").toString(); + String dataPath1 = "user_data/dummy_tee1.gor"; + String dataPath2 = "user_data/dummy_tee2.gor"; + Files.createDirectories(Path.of(gorRoot).resolve("result_cache")); + + Files.copy(Path.of("../tests/data/gor/genes.gor"), Path.of(gorRoot).resolve("genes.gor")); + + String result = runGorPipeServer(String.format( + "gor genes.gor " + + "| tee -h >(segspan -maxseg 10000 | join -segseg genes.gor | write -link user_data/tee1.gor s3data://shared/%s) " + + "| tee -h >(segspan -maxseg 5000 | join -segseg genes.gor | write -link user_data/tee2.gor s3data://shared/%s) " + + "| segspan -maxseg 10000\n", + dataPath1, dataPath2), gorRoot, securityContext); + + Assert.assertEquals("Incorrect group size", 234921, result.split("\n").length); + + S3SharedSourceProvider provider = new S3ProjectSharedSourceProvider(); + provider.setConfig(ConfigManager.getPrefixConfig("gor", GorDriverConfig.class)); + try (DataSource source = getDataSourceFromProvider(provider, dataPath1, Credentials.OwnerType.System, "some_env")) { + source.delete(); + } + try (DataSource source = getDataSourceFromProvider(provider, dataPath2, Credentials.OwnerType.System, "some_env")) { + source.delete(); + } + } + private DataSource getDataSourceFromProvider(S3SharedSourceProvider provider, String relativePath, Credentials.OwnerType ownerType, String owner) throws IOException { SourceReference sourceReference = new SourceReferenceBuilder(provider.getSharedUrlPrefix() + relativePath) diff --git a/gortools/src/test/java/gorsat/UTestGorTee.java b/gortools/src/test/java/gorsat/UTestGorTee.java index 943a5e98e..58c34a066 100644 --- a/gortools/src/test/java/gorsat/UTestGorTee.java +++ b/gortools/src/test/java/gorsat/UTestGorTee.java @@ -53,7 +53,7 @@ public void setUpTest() throws IOException { @Test public void testGorTee() throws IOException { - Path p = Files.createTempFile("gortee", "test"); + Path p = workDir.getRoot().toPath().resolve("testGorTee.gor"); String gorcmd = "gor ../tests/data/gor/dbsnp_test.gor | top 5 | tee >(group genome -count | write " + p.toAbsolutePath().toString() + ")"; TestUtils.runGorPipeCount(gorcmd); @@ -65,7 +65,7 @@ public void testGorTee() throws IOException { @Test public void testNorTee() throws IOException { - Path p = Files.createTempFile("nortee", "test"); + Path p = workDir.getRoot().toPath().resolve("testNorTee.nor"); String gorcmd = "nor " + gorFile.getCanonicalPath() + " | top 15 | tee >(group -count| write " + p.toAbsolutePath().toString() + ")"; TestUtils.runGorPipeCount(gorcmd); @@ -77,8 +77,8 @@ public void testNorTee() throws IOException { @Test public void testDoesNotSwallow() throws IOException { - final File teeOut = workDir.newFile("teeOut.gor"); - final File otherOut = workDir.newFile("outherOut.gor"); + final File teeOut = workDir.newFile("testDoesNotSwallow_teeOut.gor"); + final File otherOut = workDir.newFile("testDoesNotSwallow_outherOut.gor"); final String query = "gor ../tests/data/gor/dbsnp_test.gor | tee >(write " + teeOut.getAbsolutePath() + " ) | write " + otherOut.getAbsolutePath(); TestUtils.runGorPipe(query); diff --git a/model/src/main/java/org/gorpipe/gor/driver/providers/stream/sources/mdr/MdrConfiguration.java b/model/src/main/java/org/gorpipe/gor/driver/providers/stream/sources/mdr/MdrConfiguration.java index d4eb0b8e8..d9dc455b9 100644 --- a/model/src/main/java/org/gorpipe/gor/driver/providers/stream/sources/mdr/MdrConfiguration.java +++ b/model/src/main/java/org/gorpipe/gor/driver/providers/stream/sources/mdr/MdrConfiguration.java @@ -84,7 +84,7 @@ static HashMap loadMdrConfigurations(MdrConfiguration HashMap mdrConfigurationsMap = new HashMap<>(); mdrConfigurationsMap.put(defaultConfig.mdrServerName(), defaultConfig); - final String MDR_CREDENTIALS_PATH = System.getProperty("gor.mdr.credentials"); + final String MDR_CREDENTIALS_PATH = System.getProperty("gor.mdr.credentials", "/mnt/csa/config/gor-mdr-config.tsv"); if (!Strings.isNullOrEmpty(MDR_CREDENTIALS_PATH)) { try { diff --git a/model/src/main/java/org/gorpipe/gor/driver/providers/stream/sources/mdr/MdrServer.java b/model/src/main/java/org/gorpipe/gor/driver/providers/stream/sources/mdr/MdrServer.java index b5da032fb..31f45befc 100644 --- a/model/src/main/java/org/gorpipe/gor/driver/providers/stream/sources/mdr/MdrServer.java +++ b/model/src/main/java/org/gorpipe/gor/driver/providers/stream/sources/mdr/MdrServer.java @@ -29,6 +29,7 @@ public class MdrServer { private static final String INCLUDE_GROUPED = "include_grouped"; private static final String MDR_ENV = "env"; + /* Default configuration from env vars. Includes config that is same for all MDR servers.*/ private static final MdrConfiguration defaultConfig = ConfigManager.createPrefixConfig("gor.mdr", MdrConfiguration.class); private static HashMap mdrServers; @@ -36,10 +37,18 @@ public class MdrServer { private static final Cache documentCache = CacheBuilder.newBuilder().concurrencyLevel(4).expireAfterAccess(defaultConfig.mdrCacheDuration(), TimeUnit.MINUTES).build(); - static { - loadMdrServers(defaultConfig); + /* Lazy load the mdr servers */ + private static HashMap getMdrServers() { + if (mdrServers != null) { + loadMdrServers(defaultConfig); + } + return mdrServers; } + /** + * Populate MDR servers, by reading the MDR server config from disk. + * @param defaultConfig the default MDR server config. The default config is set directly from env vars. + */ public static void loadMdrServers(MdrConfiguration defaultConfig) { mdrServers = MdrConfiguration.loadMdrConfigurations(defaultConfig).entrySet().stream() .collect(HashMap::new, @@ -49,21 +58,21 @@ public static void loadMdrServers(MdrConfiguration defaultConfig) { public static String resolveUrl(String url) { URI uri = URI.create(url); - MdrServer server = mdrServers.get(extractMdrEnvName(uri)); + MdrServer server = getMdrServers().get(extractMdrEnvName(uri)); if (server == null) { throw new GorResourceException("Can not resolve MDR url %s, config for env %s not found.".formatted(url, extractMdrEnvName(uri)), url); } return server.resolveMdrUrl(uri); } - public static void cacheUrls(List sources) { + static void cacheUrls(List sources) { HashMap> sourcesByMdrServer = new HashMap<>(); for (SourceRef source : sources) { var mdrServerName = extractMdrEnvName(URI.create(source.file)); sourcesByMdrServer.computeIfAbsent(mdrServerName, k -> new java.util.ArrayList<>()).add(source); } for (var entry : sourcesByMdrServer.keySet()) { - mdrServers.get(entry).cacheMdrUrls(sourcesByMdrServer.get(entry)); + getMdrServers().get(entry).cacheMdrUrls(sourcesByMdrServer.get(entry)); } } From 2d53a40cfcecb8ca354ff8a43a3ba6a346c37cf4 Mon Sep 17 00:00:00 2001 From: Gisli Magnusson Date: Fri, 30 Jan 2026 10:40:52 +0000 Subject: [PATCH 2/2] fix(ENGKNOW-3126): Defaults for MDR config location (sneaked in minor improvments for tee testing). --- .../org/gorpipe/s3/shared/ITestS3Shared.java | 29 ------------------- 1 file changed, 29 deletions(-) diff --git a/drivers/src/test/java/org/gorpipe/s3/shared/ITestS3Shared.java b/drivers/src/test/java/org/gorpipe/s3/shared/ITestS3Shared.java index 2e1026b38..4e8250467 100644 --- a/drivers/src/test/java/org/gorpipe/s3/shared/ITestS3Shared.java +++ b/drivers/src/test/java/org/gorpipe/s3/shared/ITestS3Shared.java @@ -598,35 +598,6 @@ public void testWriteExplicitWrite() throws IOException { } } - @Test - public void testWriteWithTeeAndLinks() throws IOException { - String securityContext = createSecurityContext("s3data", Credentials.OwnerType.System, "some_env", S3_KEY, S3_SECRET); - String gorRoot = Path.of(workDir.getRoot().toString(), "some_project").toString(); - String dataPath1 = "user_data/dummy_tee1.gor"; - String dataPath2 = "user_data/dummy_tee2.gor"; - Files.createDirectories(Path.of(gorRoot).resolve("result_cache")); - - Files.copy(Path.of("../tests/data/gor/genes.gor"), Path.of(gorRoot).resolve("genes.gor")); - - String result = runGorPipeServer(String.format( - "gor genes.gor " + - "| tee -h >(segspan -maxseg 10000 | join -segseg genes.gor | write -link user_data/tee1.gor s3data://shared/%s) " + - "| tee -h >(segspan -maxseg 5000 | join -segseg genes.gor | write -link user_data/tee2.gor s3data://shared/%s) " + - "| segspan -maxseg 10000\n", - dataPath1, dataPath2), gorRoot, securityContext); - - Assert.assertEquals("Incorrect group size", 234921, result.split("\n").length); - - S3SharedSourceProvider provider = new S3ProjectSharedSourceProvider(); - provider.setConfig(ConfigManager.getPrefixConfig("gor", GorDriverConfig.class)); - try (DataSource source = getDataSourceFromProvider(provider, dataPath1, Credentials.OwnerType.System, "some_env")) { - source.delete(); - } - try (DataSource source = getDataSourceFromProvider(provider, dataPath2, Credentials.OwnerType.System, "some_env")) { - source.delete(); - } - } - private DataSource getDataSourceFromProvider(S3SharedSourceProvider provider, String relativePath, Credentials.OwnerType ownerType, String owner) throws IOException { SourceReference sourceReference = new SourceReferenceBuilder(provider.getSharedUrlPrefix() + relativePath)