diff --git a/gortools/src/main/java/gorsat/process/GorJavaUtilities.java b/gortools/src/main/java/gorsat/process/GorJavaUtilities.java index b6db2c0c..0a03a605 100644 --- a/gortools/src/main/java/gorsat/process/GorJavaUtilities.java +++ b/gortools/src/main/java/gorsat/process/GorJavaUtilities.java @@ -579,12 +579,9 @@ public static String verifyLinkFileLastModified(ProjectContext projectContext, S public static void writeDictionaryFromMeta(FileReader fileReader, String outfolderpath, String dictionarypath) throws IOException { FileReader localFileReader = fileReader; - if (PathUtils.isLocal(outfolderpath)) { - // Force meta data update on the parent (solves issue with NFS sycn) - try (Stream paths = fileReader.list(PathUtils.getParent(outfolderpath))) { - // Intentionally empty. - } - } + + fileReader.updateFileSystemMetaData(outfolderpath); + try (Stream metapathstream = localFileReader.list(outfolderpath); Writer dictionarypathwriter = new OutputStreamWriter(localFileReader.getOutputStream(dictionarypath))) { var metaList = metapathstream.parallel().filter(p -> DataUtil.isMeta(p)).map(p -> GorMeta.createAndLoad(localFileReader, p)).collect(Collectors.toList()); diff --git a/gortools/src/main/scala/gorsat/InputSources/Gorif.scala b/gortools/src/main/scala/gorsat/InputSources/Gorif.scala index 36e5695c..ea09c0dd 100644 --- a/gortools/src/main/scala/gorsat/InputSources/Gorif.scala +++ b/gortools/src/main/scala/gorsat/InputSources/Gorif.scala @@ -19,7 +19,7 @@ class Gorif extends InputSourceInfo("GORIF", // Filter out filepath if not exists val nonExistentFiles = iargs.filter(arg => !CommandParseUtilities.isNestedCommand(arg) && - !context.getSession.getProjectContext.getFileReader.exists(arg)).toList + !context.getSession.getProjectContext.getFileReader.existsWithMetaDataUpdate(arg)).toList val checkedIargs = iargs.filterNot(nonExistentFiles.contains) if (checkedIargs.isEmpty) { diff --git a/gortools/src/main/scala/gorsat/InputSources/Nor.scala b/gortools/src/main/scala/gorsat/InputSources/Nor.scala index 8663c48a..f62fb788 100644 --- a/gortools/src/main/scala/gorsat/InputSources/Nor.scala +++ b/gortools/src/main/scala/gorsat/InputSources/Nor.scala @@ -189,7 +189,7 @@ object Nor // Filter out filepath if not exists val nonExistentFiles = iargs.filter(arg => !CommandParseUtilities.isNestedCommand(arg) && - !context.getSession.getProjectContext.getFileReader.exists(arg)).toList + !context.getSession.getProjectContext.getFileReader.existsWithMetaDataUpdate(arg)).toList val checkedIargs = iargs.filterNot(nonExistentFiles.contains) if (checkedIargs.isEmpty) { diff --git a/model/src/main/java/org/gorpipe/gor/model/FileReader.java b/model/src/main/java/org/gorpipe/gor/model/FileReader.java index 7149d1b8..ac0d7dfd 100644 --- a/model/src/main/java/org/gorpipe/gor/model/FileReader.java +++ b/model/src/main/java/org/gorpipe/gor/model/FileReader.java @@ -28,6 +28,7 @@ import org.gorpipe.gor.driver.meta.SourceReference; import org.gorpipe.gor.driver.providers.stream.sources.StreamSource; import org.gorpipe.gor.driver.utils.LinkFile; +import org.gorpipe.gor.table.util.PathUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -363,4 +364,42 @@ public FileReader unsecure() { public Stream prepareSources(Stream sources) { return sources; } + + /** + * Update the file system metadata for the given path/file. + * NOTE: This can be expensive, so it should be used with care. This is only needed for + * (shared)filesystems (not object storage). + * @param path + * @throws IOException + */ + public void updateFileSystemMetaData(String path) throws IOException { + if (PathUtils.isLocal(path)) { + // Force meta data update on the parent (solves issue with NFS sycn) + try (Stream paths = this.list(PathUtils.getParent(path))) { + // Intentionally empty. + } + } + } + + /** + * Check if a file exists, and if not, update the file system metadata for the given path. + * This is useful when working with files that may have been recently created or modified. + * + * @param path The path to check for existence. + * @return true if the file exists, false otherwise. + */ + public boolean existsWithMetaDataUpdate(String path) { + if (exists(path)) { + return true; + } else { + try { + // If the file does not exist, we try to update the metadata. + updateFileSystemMetaData(path); + } catch (IOException e) { + log.warn("Could not update file system metadata for path: {}", path, e); + return false; + } + return exists(path); + } + } }