diff --git a/gortools/src/main/java/org/gorpipe/gor/manager/BucketManager.java b/gortools/src/main/java/org/gorpipe/gor/manager/BucketManager.java index 01087cda..028bd3bf 100644 --- a/gortools/src/main/java/org/gorpipe/gor/manager/BucketManager.java +++ b/gortools/src/main/java/org/gorpipe/gor/manager/BucketManager.java @@ -42,6 +42,7 @@ import org.slf4j.LoggerFactory; import java.io.IOException; +import java.nio.file.Path; import java.time.Duration; import java.util.*; import java.util.function.Function; @@ -456,7 +457,7 @@ private int doBucketize(TableLock bucketizeLock, BucketPackLevel packLevel, int private void doBucketizeForBucketDir(DictionaryTable tempTable, String bucketDir, Map> newBucketsMap, int maxBucketCount) throws IOException { Map> newBucketsMapForBucketDir = newBucketsMap.keySet().stream() - .filter(p -> PathUtils.getParent(p).equals(PathUtils.stripTrailingSlash(bucketDir))) + .filter(p -> PathUtils.pathEquals(PathUtils.getParent(p), bucketDir)) .collect(Collectors.toMap(Function.identity(), newBucketsMap::get)); // Create the bucket files diff --git a/gortools/src/test/java/org/gorpipe/gor/manager/UTestBucketManager.java b/gortools/src/test/java/org/gorpipe/gor/manager/UTestBucketManager.java index a7ac5b99..899c2c10 100644 --- a/gortools/src/test/java/org/gorpipe/gor/manager/UTestBucketManager.java +++ b/gortools/src/test/java/org/gorpipe/gor/manager/UTestBucketManager.java @@ -665,7 +665,7 @@ public void testBucketFolders() throws Exception { buckets = table.filter().get().stream().map(l -> l.getBucket()).distinct().collect(Collectors.toList()); List bucketFolders = buckets.stream().map(l -> PathUtils.getParent(l)).distinct().collect(Collectors.toList()); Assert.assertEquals("Incorrect number of bucket folders", 1, bucketFolders.size()); - Assert.assertEquals("Incorrect bucket folder(s)", buc.pickBucketDir(), bucketFolders.get(0)); + Assert.assertEquals("Incorrect bucket folder(s)", Path.of(buc.pickBucketDir()), Path.of(bucketFolders.get(0))); for (String bucket : buckets) { Assert.assertTrue("Bucket does not exists", Files.exists(Path.of(PathUtils.resolve(table.getRootPath(), bucket)))); } @@ -839,7 +839,9 @@ private void testBucketDirsHelper(BucketManager buc, GorDictionaryTable table, L buckets = table.filter().get().stream().map(l -> l.getBucket()).distinct().collect(Collectors.toList()); List createdBucketFolders = buckets.stream().map(p -> PathUtils.getParent(p)).distinct().collect(Collectors.toList()); Assert.assertEquals("Incorrect number of bucket folders", bucketDirs.size(), createdBucketFolders.size()); - Assert.assertEquals("Incorrect bucket folder(s)", new TreeSet(bucketDirs), new TreeSet(createdBucketFolders)); + Assert.assertEquals("Incorrect bucket folder(s)", + new TreeSet(bucketDirs.stream().map(s -> Path.of(s)).toList()), + new TreeSet(createdBucketFolders.stream().map(s -> Path.of(s)).toList())); for (String bucket : buckets) { Assert.assertTrue(String.format("Bucket %s does not exists", PathUtils.resolve(table.getRootPath(), bucket)), Files.exists(Path.of(PathUtils.resolve(table.getRootPath(), bucket)))); diff --git a/model/src/main/java/org/gorpipe/gor/driver/providers/stream/sources/file/FileSourceType.java b/model/src/main/java/org/gorpipe/gor/driver/providers/stream/sources/file/FileSourceType.java index 391e8123..536db3cd 100644 --- a/model/src/main/java/org/gorpipe/gor/driver/providers/stream/sources/file/FileSourceType.java +++ b/model/src/main/java/org/gorpipe/gor/driver/providers/stream/sources/file/FileSourceType.java @@ -40,9 +40,6 @@ public PRIORITY getPriority() { @Override public boolean match(String file) { // TODO: Until we get better matching strategy we must exclude mem here. - // TODO: Should check if file (as commented out), but a lot code seems to pass various - // non file elements so simply match everything (like the former impl). - // We should be the last matcher anyway. - return !DataUtil.isMem(file); //&& (file.startsWith("file:") || !file.contains(":")); + return !DataUtil.isMem(file) && (file.startsWith("file:") || !file.contains(":/")); } } diff --git a/model/src/main/java/org/gorpipe/gor/table/util/PathUtils.java b/model/src/main/java/org/gorpipe/gor/table/util/PathUtils.java index bc196e84..1b38e2e5 100644 --- a/model/src/main/java/org/gorpipe/gor/table/util/PathUtils.java +++ b/model/src/main/java/org/gorpipe/gor/table/util/PathUtils.java @@ -261,7 +261,7 @@ public static URI toRealPath(URI uri) { public static String getParent(String path) { String p = stripTrailingSlash(path); var idx = p.lastIndexOf("/"); - return idx >= 0 ? p.substring(0,idx) : ""; + return idx >= 0 ? p.substring(0,idx+1) : ""; } public static String getFileName(String path) { @@ -343,4 +343,14 @@ public static Path getTempFilePath(Path filePath) { return tempFileName; } + public static boolean pathEquals(String path1, String path2) { + if (path1 == null && path2 == null) { + return true; + } + if (path1 == null || path2 == null) { + return false; + } + return Path.of(path1).normalize().equals(Path.of(path2).normalize()); + } + } diff --git a/model/src/test/java/org/gorpipe/gor/driver/UTestPluggableGorDriver.java b/model/src/test/java/org/gorpipe/gor/driver/UTestPluggableGorDriver.java index be6d46ad..7583a096 100644 --- a/model/src/test/java/org/gorpipe/gor/driver/UTestPluggableGorDriver.java +++ b/model/src/test/java/org/gorpipe/gor/driver/UTestPluggableGorDriver.java @@ -23,8 +23,10 @@ package org.gorpipe.gor.driver; import gorsat.TestUtils; +import org.gorpipe.exceptions.GorResourceException; import org.gorpipe.gor.driver.meta.DataType; import org.gorpipe.gor.driver.meta.SourceReference; +import org.gorpipe.gor.driver.providers.stream.sources.file.FileSource; import org.gorpipe.gor.model.GenomicIterator; import org.junit.Assert; import org.junit.Test; @@ -64,4 +66,35 @@ public void noGuice() throws IOException { Assert.assertNotNull(iterator); } + @Test + public void testResolveDataSourceFile() throws IOException { + String fileName = "testfile.gor"; + DataSource source = gorDriver.resolveDataSource(new SourceReference(fileName)); + Assert.assertTrue(source instanceof FileSource); + Assert.assertEquals(fileName, source.getName()); + } + + @Test + public void testResolveDataSourceFileSchema() throws IOException { + String fileName = "testfile.gor"; + DataSource source = gorDriver.resolveDataSource(new SourceReference("file://" + fileName)); + Assert.assertTrue(source instanceof FileSource); + Assert.assertEquals(fileName, source.getName()); + } + + @Test + public void testResolveDataSourceS3Schema() throws IOException { + String fileName = "s3://testfile.gor"; + DataSource source = gorDriver.resolveDataSource(new SourceReference(fileName)); + Assert.assertEquals(fileName, source.getName()); + } + + @Test + public void testResolveDataSourceUnknownSchema() throws IOException { + String fileName = "bla://testfile" + DataType.GOR.suffix; + Assert.assertThrows(GorResourceException.class, () -> { + gorDriver.resolveDataSource(new SourceReference(fileName)); + }); + } + } \ No newline at end of file