feat(ENGKNOW-2891): gor versioned link file metadata support

documentation/src/command/TSVAPPEND.rst

@@ -14,7 +14,7 @@ Usage
 
 .. code-block:: gor
 
-	nor ... | tsvappend <file name> [-noheader] [-prefix <prefix>] [-link <link path>]
+	nor ... | tsvappend <file name> [-noheader] [-prefix <prefix>] [-link <link path> [-lmeta <link meta>]
 
 Options
 =======
@@ -25,13 +25,13 @@ Options
 +-------------------+-----------------------------------------------------------------+
 | ``-noheader``     | Don't write a header lines.  Not valid with gor/gorz/nor/norz.  |
 +-------------------+-----------------------------------------------------------------+
-| ``-link <link>``  | Writes a link file pointing to the the <file name>.             |
-|                   |                                                                 |
-+-------------------+-----------------------------------------------------------------+
-| ``-vlink <link>`` | Writes a versioned link file pointing to the the <file name>.   |
+| ``-link <link>``  | Writes a versioned link file pointing to the the <file name>.   |
 |                   | The <file name> should not be overwritten if it has previously  |
 |                   | been used in a link file.                                       |
 +-------------------+-----------------------------------------------------------------+
+| ``-linkmeta <m>`` | Writes <meta> as meta data to the <link>.  <meta> is string for |
+|                   | of comma separated key=value elements.                          |
++-------------------+-----------------------------------------------------------------+
 
 Examples
 ========

documentation/src/command/WRITE.rst

@@ -62,13 +62,13 @@ Options
 | ``-card '<cols>'``| Calculate cardinality of columns in '<cols>' and adds to the    |
 |                   | outputs meta data.                                              |
 +-------------------+-----------------------------------------------------------------+
-| ``-link <link>``  | Writes a link file pointing to the the <file name>.             |
-|                   |                                                                 |
-+-------------------+-----------------------------------------------------------------+
-| ``-vlink <link>`` | Writes a versioned link file pointing to the the <file name>.   |
+| ``-link <link>``  | Writes a versioned link file pointing to the the <file name>.   |
 |                   | The <file name> should not be overwritten if it has previously  |
 |                   | been used in a link file.                                       |
 +-------------------+-----------------------------------------------------------------+
+| ``-linkmeta <m>`` | Writes <meta> as meta data to the <link>.  <meta> is string for |
+|                   | of comma separated key=value elements.                          |
++-------------------+-----------------------------------------------------------------+
 
 Examples
 ========

drivers/src/test/java/org/gorpipe/s3/table/ITestS3Table.java

@@ -3,7 +3,9 @@
 import gorsat.TestUtils;
 import org.gorpipe.base.security.BundledCredentials;
 import org.gorpipe.base.security.Credentials;
+import org.gorpipe.gor.driver.linkfile.LinkFile;
 import org.gorpipe.gor.driver.meta.DataType;
+import org.gorpipe.gor.driver.providers.stream.sources.file.FileSource;
 import org.gorpipe.gor.model.DriverBackedFileReader;
 import org.gorpipe.gor.table.dictionary.DictionaryTable;
 import org.gorpipe.gor.table.lock.NoTableLock;
@@ -224,7 +226,7 @@ public void testBucketizeLocalTableS3DataS3DataBucketsRelative() throws IOExcept
 
             String localBucketFile = PathUtils.resolve(table.getRootPath(), DataUtil.toFile( bucket, DataType.LINK));
 
-            Assert.assertEquals("s3data://project/" +  bucket + "\n", Files.readString(Path.of(localBucketFile)));
+            Assert.assertEquals("s3data://project/" +  bucket, LinkFile.load(new FileSource(localBucketFile)).getLatestEntryUrl());
 
             String[] bucketResult = runGorPipeServer("gor " + localBucketFile,
                     workDirPath.resolve("some_project").toString(), fileReader.getSecurityContext()).split("\n");

gortools/src/main/scala/gorsat/Analysis/ForkWrite.scala

@@ -23,19 +23,21 @@
 package gorsat.Analysis
 
 import java.util.zip.Deflater
-import gorsat.Commands.{Analysis, Output, RowHeader}
+import gorsat.Commands.{Analysis, CommandParseUtilities, Output, RowHeader}
 import gorsat.Outputs.OutFile
 import org.apache.commons.io.FilenameUtils
+import org.apache.commons.lang3.StringUtils
 import org.gorpipe.exceptions.GorResourceException
 import org.gorpipe.gor.binsearch.GorIndexType
-import org.gorpipe.gor.driver.linkfile.LinkFile
+import org.gorpipe.gor.driver.linkfile.{LinkFile, LinkFileEntryV1}
 import org.gorpipe.gor.driver.meta.DataType
 import org.gorpipe.gor.driver.providers.stream.sources.StreamSource
 import org.gorpipe.gor.model.{DriverBackedFileReader, GorMeta, GorOptions, Row}
 import org.gorpipe.gor.session.{GorSession, ProjectContext}
 import org.gorpipe.gor.table.util.PathUtils
 import org.gorpipe.gor.util.DataUtil
 import org.gorpipe.model.gor.RowObj
+import org.gorpipe.util.Strings
 
 import java.util.UUID
 import scala.collection.mutable
@@ -57,7 +59,7 @@ case class OutputOptions(remove: Boolean = false,
                             writeMeta: Boolean = true,
                             cardCol: String = null,
                             linkFile: String = "",
-                            linkFileVersion: Int = 1,
+                            linkFileMeta: String = "",
                             command: String = null,
                             infer: Boolean = false,
                             maxseg: Boolean = false
@@ -274,17 +276,17 @@ case class ForkWrite(forkCol: Int,
 
     if (useFork) {
       forkMap.values.foreach(sh => {
-        val (linkFile, linkFileContent, linkFileVersion) = extractLink(sh.fileName)
+        val (linkFile, linkFileUrl, linkFileMeta, linkFileInfo) = extractLink(sh.fileName)
 
         if (linkFile.nonEmpty) {
-          writeLinkFile(linkFile, linkFileContent, linkFileVersion)
+          writeLinkFile(linkFile, linkFileUrl, linkFileMeta, linkFileInfo)
         }
       })
     } else {
-      val (linkFile, linkFileContent, linkFileVersion) = extractLink(fullFileName, options.linkFile, options.linkFileVersion)
+      val (linkFile, linkFileUrl, linkFileMeta, linkFileInfo) = extractLink(fullFileName, options.linkFile, options.linkFileMeta)
 
       if (linkFile.nonEmpty) {
-        writeLinkFile(linkFile, linkFileContent, linkFileVersion, getMd5)
+        writeLinkFile(linkFile, linkFileUrl, linkFileMeta, getMd5, linkFileInfo)
       }
     }
   }
@@ -298,38 +300,51 @@ case class ForkWrite(forkCol: Int,
     }
   }
 
-  private def extractLink(fileName: String, optLinkFile: String = "", optLinkFileVersion: Int = 0) : (String, String, Int) = {
+  private def extractLink(fileName: String, optLinkFile: String = "", optLinkFileMeta: String = "") : (String, String, String, String) = {
     var linkFile = optLinkFile
-    var linkFileVersion = optLinkFileVersion
     var linkFileContent = ""
     if (fileName.nonEmpty) {
       if (linkFile.isEmpty)  {
         val dataSource = session.getProjectContext.getFileReader.resolveUrl(fileName, true)
         if (dataSource != null && dataSource.forceLink()) {
           linkFile = dataSource.getProjectLinkFile
           linkFileContent = dataSource.getProjectLinkFileContent
-          linkFileVersion = 0
         }
       } else {
         linkFileContent = PathUtils.resolve(session.getProjectContext.getProjectRoot, fileName)
       }
     }
-    (linkFile,linkFileContent,linkFileVersion)
+
+    var linkFileMeta = ""
+    var linkFileInfo = ""
+    if (!Strings.isNullOrEmpty(optLinkFileMeta)) {
+      for (s <- CommandParseUtilities.quoteSafeSplit(StringUtils.strip(optLinkFileMeta, "\"\'"), ',')) {
+        val l = s.trim
+        if (l.startsWith(LinkFileEntryV1.ENTRY_INFO_KEY)) {
+          linkFileInfo =  StringUtils.strip(l.substring(LinkFileEntryV1.ENTRY_INFO_KEY.length  + 1), "\"\'")
+        } else {
+          linkFileMeta += "## " + l + "\n"
+        }
+      }
+    }
+
+    (linkFile, linkFileContent, linkFileMeta, linkFileInfo)
   }
 
   private def writeLinkFile(linkFilePath: String, linkFileContent: String,
-                            linkFileVersion: Int = 0, md5: String = null) : Unit = {
-    val linkFileToWrite = LinkFile.validateAndUpdateLinkFileName(linkFilePath, linkFileVersion)
+                            linkFileMeta: String = "", md5: String = null, linkFileInfo: String = null) : Unit = {
+    val linkFileToWrite = LinkFile.validateAndUpdateLinkFileName(linkFilePath)
 
      // Validate that we can write to the location (skip link extension as writing links is always forbidden).
     session.getProjectContext.getFileReader.resolveUrl(FilenameUtils.removeExtension(linkFileToWrite), true)
 
     // Use the nonsecure driver file reader as this is an exception from the write no links rule.
     val fileReader = new DriverBackedFileReader(session.getProjectContext.getFileReader.getSecurityContext,
-      session.getProjectContext.getProjectRoot)
+      session.getProjectContext.getProjectRoot, session.getProjectContext.getFileReader.getQueryTime)
 
-    LinkFile.load(fileReader.resolveUrl(linkFileToWrite, true).asInstanceOf[StreamSource], linkFileVersion)
-      .appendEntry(linkFileContent, md5, fileReader)
-      .save()
+    LinkFile.load(fileReader.resolveUrl(linkFileToWrite, true).asInstanceOf[StreamSource])
+      .appendMeta(linkFileMeta)
+      .appendEntry(linkFileContent, md5, linkFileInfo, fileReader)
+      .save(session.getProjectContext.getFileReader.getQueryTime)
   }
 }

gortools/src/main/scala/gorsat/Commands/TsvAppend.scala

@@ -32,7 +32,7 @@ import org.gorpipe.gor.util.StringUtil
 
 
 class TsvAppend extends CommandInfo("TSVAPPEND",
-  CommandArguments("-noheader", "-prefix -link -vlink", 0),
+  CommandArguments("-noheader", "-prefix -link -linkmeta", 0),
   CommandOptions(gorCommand = false, norCommand = true, verifyCommand = true)) {
   override def processArguments(context: GorContext, argString: String, iargs: Array[String], args: Array[String], executeNor: Boolean, forcedInputHeader: String): CommandParsingResult = {
 
@@ -72,16 +72,8 @@ class TsvAppend extends CommandInfo("TSVAPPEND",
       else prefixFile = Option(prfx)
     }
 
-    if (hasOption(args, "-link") && hasOption(args, "-vlink")) {
-      throw new GorParsingException("Options -link and -vlink are mutually exclusive")
-    }
-    val (link, linkVersion) = if (hasOption(args, "-link")) {
-      (stringValueOfOption(args, "-link"), 0)
-    } else if (hasOption(args, "-vlink")) {
-      (stringValueOfOption(args, "-vlink"), 1)
-    } else {
-      ("", 0)
-    }
+    val linkOpt = if (hasOption(args, "-link")) stringValueOfOption(args, "-link") else ""
+    val linkMetaOpt = if (hasOption(args, "-linkmeta")) stringValueOfOption(args, "-linkmeta") else ""
 
     val fixedHeader = forcedInputHeader.split("\t").slice(0, 2).mkString("\t")
 
@@ -95,8 +87,8 @@ class TsvAppend extends CommandInfo("TSVAPPEND",
           prefix=prefix,
           prefixFile=prefixFile,
           skipHeader=skipHeader,
-          linkFile=link,
-          linkFileVersion=linkVersion,
+          linkFile=linkOpt,
+          linkFileMeta=linkMetaOpt,
           command=argString
         )
       ),

gortools/src/main/scala/gorsat/Commands/Write.scala

@@ -34,7 +34,7 @@ import org.gorpipe.gor.util.DataUtil
 
 
 class Write extends CommandInfo("WRITE",
-  CommandArguments("-r -c -m -inferschema -maxseg -noheader", "-d -f -i -t -l -tags -card -prefix -link -vlink", 0),
+  CommandArguments("-r -c -m -inferschema -maxseg -noheader", "-d -f -i -t -l -tags -card -prefix -link -linkmeta", 0),
   CommandOptions(gorCommand = true, norCommand = true, verifyCommand = true)) {
   override def processArguments(context: GorContext, argString: String, iargs: Array[String], args: Array[String], executeNor: Boolean, forcedInputHeader: String): CommandParsingResult = {
 
@@ -63,16 +63,8 @@ class Write extends CommandInfo("WRITE",
     md5 = hasOption(args, "-m")
     if (hasOption(args, "-l")) compressionLevel = stringValueOfOptionWithErrorCheck(args, "-l", Array("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")).toInt
 
-    if (hasOption(args, "-link") && hasOption(args, "-vlink")) {
-      throw new GorParsingException("Options -link and -vlink are mutually exclusive")
-    }
-    val (link, linkVersion) = if (hasOption(args, "-link")) {
-      (stringValueOfOption(args, "-link"), 0)
-    } else if (hasOption(args, "-vlink")) {
-      (stringValueOfOption(args, "-vlink"), 1)
-    } else {
-      ("", 0)
-    }
+    val linkOpt = if (hasOption(args, "-link")) stringValueOfOption(args, "-link") else ""
+    val linkMetaOpt = if (hasOption(args, "-linkmeta")) stringValueOfOption(args, "-linkmeta") else ""
 
     if(fileName.isEmpty && useFolder.isEmpty) throw new GorResourceException("No file or folder specified","");
 
@@ -148,8 +140,8 @@ class Write extends CommandInfo("WRITE",
           useFolder,
           skipHeader,
           cardCol = card,
-          linkFile = link,
-          linkFileVersion = linkVersion,
+          linkFile = linkOpt,
+          linkFileMeta = linkMetaOpt,
           command = argString,
           infer = infer,
           maxseg = maxseg

gortools/src/main/scala/gorsat/process/PipeInstance.scala

@@ -37,7 +37,7 @@ import gorsatGorIterator.{MemoryMonitorUtil, gorsatGorIterator}
 import process.GorJavaUtilities.CmdParams
 import process.GorPipe.brsConfig
 import org.gorpipe.exceptions.{GorParsingException, GorResourceException, GorSystemException, GorUserException}
-import org.gorpipe.gor.model.{DriverBackedFileReader, FileReader, GenomicIterator}
+import org.gorpipe.gor.model.GenomicIterator
 import org.gorpipe.gor.monitor.GorMonitor
 import org.gorpipe.gor.session.{GorContext, GorSession, ProjectContext}
 import org.gorpipe.gor.util.{CommandSubstitutions, StringUtil}
@@ -49,8 +49,6 @@ object PipeInstance {
 
   private val logger = LoggerFactory.getLogger(this.getClass)
 
-  val DEFAULT_REQUEST_ID: String = ""
-
   // Set the dyniterator iterator create function
   DynIterator.createGorIterator = (context: GorContext) => createGorIterator(context)
 
@@ -222,14 +220,6 @@ class PipeInstance(context: GorContext, outputValidateOrder: Boolean = false) ex
 
   def getPipeStep : Analysis = thePipeStep
 
-  def createFileReader(gorRoot: String): FileReader = {
-    if (!StringUtil.isEmpty(gorRoot)) {
-      new DriverBackedFileReader(null, gorRoot)
-    } else {
-      ProjectContext.DEFAULT_READER
-    }
-  }
-
   @Deprecated
   def subProcessArguments(pipeOptions: PipeOptions): GenomicIterator = {
     init(pipeOptions.query, pipeOptions.stdIn, "")