From 778e0e1440ef7b1858ce7850c098305456d10457 Mon Sep 17 00:00:00 2001 From: Gisli Magnusson Date: Tue, 8 Jul 2025 22:14:09 +0000 Subject: [PATCH] fix(ENGKNOW-2564): Fix dict cache error causing -Y option sometimes not to work. --- .../java/org/gorpipe/gor/manager/BucketManager.java | 5 +++-- .../main/java/org/gorpipe/gor/table/Dictionary.java | 6 +++--- .../dictionary/DefaultTableAccessOptimizer.java | 8 ++++---- .../org/gorpipe/gor/Dictionary/UTestDictionary.java | 12 ++++++++++++ 4 files changed, 22 insertions(+), 9 deletions(-) diff --git a/gortools/src/main/java/org/gorpipe/gor/manager/BucketManager.java b/gortools/src/main/java/org/gorpipe/gor/manager/BucketManager.java index a393b14c7..838c1a6a9 100644 --- a/gortools/src/main/java/org/gorpipe/gor/manager/BucketManager.java +++ b/gortools/src/main/java/org/gorpipe/gor/manager/BucketManager.java @@ -73,7 +73,7 @@ public enum BucketPackLevel { public static final String HEADER_MIN_BUCKET_SIZE_KEY = "MIN_BUCKET_SIZE"; public static final String HEADER_BUCKET_SIZE_KEY = "BUCKET_SIZE"; - // Comma seperated list of bucket dirs. Absolute or relative to the table dir. + // A comma separated list of bucket dirs. Absolute or relative to the table dir. public static final String HEADER_BUCKET_DIRS_KEY = "BUCKET_DIRS"; // Bucket dir root. Used to find the absolute bucket path if bucket dir is relative. The path @@ -81,7 +81,8 @@ public enum BucketPackLevel { public static final String HEADER_BUCKET_DIRS_LOCATION_KEY = "BUCKET_DIRS_LOCATION"; public static final String HEADER_BUCKET_MAX_BUCKETS = "BUCKET_MAX_BUCKETS"; - protected Duration gracePeriodForDeletingBuckets = Duration.ofHours(24); + protected Duration gracePeriodForDeletingBuckets = Duration.ofHours(Integer.parseInt( + System.getProperty("gor.table.buckets.delete.grace.period.hours", "24"))); // Default 24 hours. // Location of the bucket files we create (absolute or relative to rootPath). private final List bucketDirs = new ArrayList<>(); diff --git a/model/src/main/java/org/gorpipe/gor/table/Dictionary.java b/model/src/main/java/org/gorpipe/gor/table/Dictionary.java index efc29395f..d89b79bfb 100644 --- a/model/src/main/java/org/gorpipe/gor/table/Dictionary.java +++ b/model/src/main/java/org/gorpipe/gor/table/Dictionary.java @@ -196,11 +196,11 @@ public DictionaryLine[] getSources(Set tags, boolean allowBucketAccess, final DictionaryLine[] toReturn; final Set badTags = new HashSet<>(); if (this.useCache) { - final String orderedTags = orderTags(tags); - toReturn = tagsToListCache.compute(orderedTags, (key, list) -> list == null ? generateList(tags, allowBucketAccess, badTags) : list); + final String inputKey = orderTags(tags) + ":" + allowBucketAccess; + toReturn = tagsToListCache.compute(inputKey, (key, list) -> list == null ? generateList(tags, allowBucketAccess, badTags) : list); // Removing the entry from cache due to invalid tags should be separate from throwing the data exception if (badTags.size() > 0) { - tagsToListCache.remove(orderedTags); + tagsToListCache.remove(inputKey); } } else { toReturn = generateList(tags, allowBucketAccess, badTags); diff --git a/model/src/main/java/org/gorpipe/gor/table/dictionary/DefaultTableAccessOptimizer.java b/model/src/main/java/org/gorpipe/gor/table/dictionary/DefaultTableAccessOptimizer.java index c6bce4d45..a739fbbd3 100644 --- a/model/src/main/java/org/gorpipe/gor/table/dictionary/DefaultTableAccessOptimizer.java +++ b/model/src/main/java/org/gorpipe/gor/table/dictionary/DefaultTableAccessOptimizer.java @@ -55,7 +55,7 @@ synchronized public List getOptimizedEntries(Set tags, List result; final Set badTags = new HashSet<>(); if (USE_CACHE) { - String key = findKey(tags, allowBucketAccess, isSilentTagFilter); + String key = findKey(tags, allowBucketAccess); result = tagsToListCache.getIfPresent(key); if (result == null) { result = calcOptimizeFileList(tags, allowBucketAccess, badTags); @@ -83,11 +83,11 @@ synchronized public Collection getBucketDeletedFiles(String bucket) { return this.bucketHasDeletedFile.get(bucket); } - private static String findKey(Collection tags, boolean allowBucketAccess, boolean isSilentTagFilter) { - if (tags == null) return ":" + allowBucketAccess + ":" + isSilentTagFilter; + private static String findKey(Collection tags, boolean allowBucketAccess) { + if (tags == null) return ":" + allowBucketAccess; final String[] stringsAsArray = tags.toArray(new String[0]); Arrays.sort(stringsAsArray); - return String.join(",", stringsAsArray) + ":" + allowBucketAccess + ":" + isSilentTagFilter; + return String.join(",", stringsAsArray) + ":" + allowBucketAccess; } private void throwBadTagException(Set badTags) { diff --git a/model/src/test/java/org/gorpipe/gor/Dictionary/UTestDictionary.java b/model/src/test/java/org/gorpipe/gor/Dictionary/UTestDictionary.java index 80a72f537..77aad1f25 100644 --- a/model/src/test/java/org/gorpipe/gor/Dictionary/UTestDictionary.java +++ b/model/src/test/java/org/gorpipe/gor/Dictionary/UTestDictionary.java @@ -415,4 +415,16 @@ public void testReadOfFileInBucketWithDeletedFiles() throws Exception { Assert.assertTrue(res1String[0].endsWith("/bucket2 null null null -1 [tag1000, tagI, tagJ, tagK, tagL, tagL2]")); } + + @Test + public void testDictionaryBucketAccess() throws IOException { + File gordDict = workDir.newFile("testDictionaryWithNoBucketAccess.gord"); + FileUtils.write(gordDict, gort1, (Charset) null); + + Dictionary dict = getDictionary(gordDict.getPath(), "."); + + Assert.assertEquals(1, dict.getSources(new HashSet<>(Arrays.asList("tagG", "tagH")), true, false).length); + Assert.assertEquals(2, dict.getSources(new HashSet<>(Arrays.asList("tagG", "tagH")), false, false).length); + } + } \ No newline at end of file