Skip to content

fix(ENGKNOW-2564): Fix dict cache error #59

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
gmagnu merged 1 commit into from
Jul 22, 2025
Merged

fix(ENGKNOW-2564): Fix dict cache error #59

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions gortools/src/main/java/org/gorpipe/gor/manager/BucketManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,15 +73,16 @@ public enum BucketPackLevel {
public static final String HEADER_MIN_BUCKET_SIZE_KEY = "MIN_BUCKET_SIZE";
public static final String HEADER_BUCKET_SIZE_KEY = "BUCKET_SIZE";

// Comma seperated list of bucket dirs. Absolute or relative to the table dir.
// A comma separated list of bucket dirs. Absolute or relative to the table dir.
Comment thread
gmagnu marked this conversation as resolved.
public static final String HEADER_BUCKET_DIRS_KEY = "BUCKET_DIRS";

// Bucket dir root. Used to find the absolute bucket path if bucket dir is relative. The path
// is <bucket location>/<project relative path to bucket>/<bucket dir>
public static final String HEADER_BUCKET_DIRS_LOCATION_KEY = "BUCKET_DIRS_LOCATION";
public static final String HEADER_BUCKET_MAX_BUCKETS = "BUCKET_MAX_BUCKETS";

protected Duration gracePeriodForDeletingBuckets = Duration.ofHours(24);
protected Duration gracePeriodForDeletingBuckets = Duration.ofHours(Integer.parseInt(
System.getProperty("gor.table.buckets.delete.grace.period.hours", "24"))); // Default 24 hours.

Comment thread
gmagnu marked this conversation as resolved.
// Location of the bucket files we create (absolute or relative to rootPath).
private final List<String> bucketDirs = new ArrayList<>();
Expand Down
6 changes: 3 additions & 3 deletions model/src/main/java/org/gorpipe/gor/table/Dictionary.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -196,11 +196,11 @@ public DictionaryLine[] getSources(Set<String> tags, boolean allowBucketAccess,
final DictionaryLine[] toReturn;
final Set<String> badTags = new HashSet<>();
if (this.useCache) {
final String orderedTags = orderTags(tags);
toReturn = tagsToListCache.compute(orderedTags, (key, list) -> list == null ? generateList(tags, allowBucketAccess, badTags) : list);
final String inputKey = orderTags(tags) + ":" + allowBucketAccess;
toReturn = tagsToListCache.compute(inputKey, (key, list) -> list == null ? generateList(tags, allowBucketAccess, badTags) : list);
// Removing the entry from cache due to invalid tags should be separate from throwing the data exception
if (badTags.size() > 0) {
tagsToListCache.remove(orderedTags);
tagsToListCache.remove(inputKey);
}
} else {
toReturn = generateList(tags, allowBucketAccess, badTags);
Expand Down
8 changes: 4 additions & 4 deletions model/src/main/java/org/gorpipe/gor/table/dictionary/DefaultTableAccessOptimizer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ synchronized public List<DictionaryEntry> getOptimizedEntries(Set<String> tags,
List<DictionaryEntry> result;
final Set<String> badTags = new HashSet<>();
if (USE_CACHE) {
String key = findKey(tags, allowBucketAccess, isSilentTagFilter);
String key = findKey(tags, allowBucketAccess);
result = tagsToListCache.getIfPresent(key);
if (result == null) {
result = calcOptimizeFileList(tags, allowBucketAccess, badTags);
Expand Down Expand Up @@ -83,11 +83,11 @@ synchronized public Collection<String> getBucketDeletedFiles(String bucket) {
return this.bucketHasDeletedFile.get(bucket);
}

private static String findKey(Collection<String> tags, boolean allowBucketAccess, boolean isSilentTagFilter) {
if (tags == null) return ":" + allowBucketAccess + ":" + isSilentTagFilter;
private static String findKey(Collection<String> tags, boolean allowBucketAccess) {
if (tags == null) return ":" + allowBucketAccess;
final String[] stringsAsArray = tags.toArray(new String[0]);
Arrays.sort(stringsAsArray);
return String.join(",", stringsAsArray) + ":" + allowBucketAccess + ":" + isSilentTagFilter;
return String.join(",", stringsAsArray) + ":" + allowBucketAccess;
}

private void throwBadTagException(Set<String> badTags) {
Expand Down
12 changes: 12 additions & 0 deletions model/src/test/java/org/gorpipe/gor/Dictionary/UTestDictionary.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -415,4 +415,16 @@ public void testReadOfFileInBucketWithDeletedFiles() throws Exception {

Assert.assertTrue(res1String[0].endsWith("/bucket2 null null null -1 [tag1000, tagI, tagJ, tagK, tagL, tagL2]"));
}

@Test
public void testDictionaryBucketAccess() throws IOException {
File gordDict = workDir.newFile("testDictionaryWithNoBucketAccess.gord");
FileUtils.write(gordDict, gort1, (Charset) null);

Dictionary dict = getDictionary(gordDict.getPath(), ".");

Assert.assertEquals(1, dict.getSources(new HashSet<>(Arrays.asList("tagG", "tagH")), true, false).length);
Assert.assertEquals(2, dict.getSources(new HashSet<>(Arrays.asList("tagG", "tagH")), false, false).length);
}

}
Loading