feat(ENGKNOW-2892): gor support for working with link files

drivers/src/main/java/org/gorpipe/s3/shared/S3SharedSourceProvider.java

@@ -97,18 +97,19 @@ public S3SharedSource resolveDataSource(SourceReference sourceReference)
 
         S3SharedSource source = null;
 
-        Credentials sharedCreds = getS3DataCredentials(getService(), sourceReference.getSecurityContext());
+        BundledCredentials bundledCreds = BundledCredentials.fromSecurityContext(sourceReference.securityContext);
+        Credentials sharedCreds = getS3DataCredentials(getService(), bundledCreds );
 
         if (sharedCreds == null) {
             log.warn(String.format("No credentials found for %s. Returning emtpy source.", getService()));
             return source;
         }
 
-        String project = Path.of(sourceReference.getCommonRoot()).getFileName().toString();
-        String bucket = sharedCreds.getLookupKey();
-        String s3SecurityContext = createS3SecurityContext(sharedCreds);
+        String s3SecurityContext = createSecurityContext(createS3CredsFromShared(sharedCreds), bundledCreds);
         String relativePath = getRelativePath(sourceReference.getUrl());
 
+        String project = Path.of(sourceReference.getCommonRoot()).getFileName().toString();
+        String bucket = sharedCreds.getLookupKey();
         SourceReference s3SourceReference = createS3SourceReference(sourceReference, project, bucket, s3SecurityContext);
 
         S3Client client = getClient(s3SecurityContext, bucket);
@@ -135,8 +136,7 @@ protected String findSharedSourceLinkContent(S3SharedSource source) {
         }
     }
 
-    private Credentials getS3DataCredentials(String service, String securityContext) {
-        BundledCredentials bundledCreds = BundledCredentials.fromSecurityContext(securityContext);
+    private Credentials getS3DataCredentials(String service, BundledCredentials bundledCreds) {
         List<Credentials> credsList = bundledCreds.getCredentialsForService(service);
         return getHighestPriorityCredential(credsList, service);
     }
@@ -159,14 +159,21 @@ private Credentials getHighestPriorityCredential(List<Credentials> credsList, St
         return bestMatch;
     }
 
-    private String createS3SecurityContext(Credentials sharedCreds) {
-        BundledCredentials bundledCredentials = new BundledCredentials.Builder().addCredentials(
-                        new Credentials("s3", sharedCreds.getLookupKey(), sharedCreds.getOwnerType(),
-                                sharedCreds.getOwnerId(), sharedCreds.expires(), sharedCreds.isUserDefault(),
-                                (Map<String, String>) sharedCreds.toMap().get("credential_attributes")))
-                .build();
-        String securityContext = bundledCredentials.addToSecurityContext("");
-        return securityContext;
+    private String createSecurityContext(Credentials s3Creds, BundledCredentials bundledCreds) {
+        BundledCredentials.Builder builder = new BundledCredentials.Builder().addCredentials(s3Creds);
+        for (String services : bundledCreds.services()) {
+            for (Credentials cred : bundledCreds.getCredentialsForService(services)) {
+                builder.addCredentials(cred);
+            }
+        }
+
+        return builder.build().addToSecurityContext("");
+    }
+
+    private Credentials createS3CredsFromShared(Credentials sharedCreds) {
+        return new Credentials("s3", sharedCreds.getLookupKey(), sharedCreds.getOwnerType(),
+                sharedCreds.getOwnerId(), sharedCreds.expires(), sharedCreds.isUserDefault(),
+                (Map<String, String>) sharedCreds.toMap().get("credential_attributes"));
     }
 
     private SourceReference createS3SourceReference(SourceReference sourceReference, String project, String bucket, String securityContext) {

gorscripts/build.gradle

@@ -48,7 +48,6 @@ project(':gorscripts') {
         implementation "org.apache.commons:commons-lang3:_"
         implementation "de.tototec:de.tototec.cmdoption:_"
         implementation "com.fasterxml.jackson.core:jackson-databind:_"
-        implementation "org.reflections:reflections:_"
         implementation "com.google.guava:guava:_"
 
         runtimeOnly project(':documentation')

gorscripts/src/main/java/org/gorpipe/gor/cli/GorCLI.java

@@ -38,10 +38,8 @@
 @CommandLine.Command(name="gor",
         version="version 1.0",
         description = "Command line interface for gor query language and processes.",
-        subcommands = {QueryCommand.class, HelpCommand.class, ManagerCommand.class, IndexCommand.class,
-                CacheCommand.class, RenderCommand.class, InfoCommand.class, FolderMigratorCommand.class,
-                LinkCommand.class})
-public class GorCLI extends HelpOptions implements Runnable {
+        subcommands = {QueryCommand.class, FolderMigratorCommand.class})
+public class GorCLI extends GorExecCLI implements Runnable {
     public static void main(String[] args) {
         GorLogbackUtil.initLog("gor");
         CommandLine cmd = new CommandLine(new GorCLI());
@@ -52,11 +50,6 @@ public static void main(String[] args) {
 
     }
 
-    @CommandLine.Option(names = {"-v", "--version"},
-            versionHelp = true,
-            description = "Print version information and exits.")
-    boolean versionHelpRequested;
-
     @Override
     public void run() {
         CommandLine.usage(this, System.err);

gorscripts/src/main/java/org/gorpipe/gor/cli/query/QueryCommand.java

@@ -28,6 +28,8 @@
 import gorsat.process.PipeOptions;
 import org.gorpipe.exceptions.ExceptionUtilities;
 import org.gorpipe.exceptions.GorException;
+import org.gorpipe.gor.cli.GorCLI;
+import org.gorpipe.gor.cli.GorExecCLI;
 import org.gorpipe.gor.cli.HelpOptions;
 import org.gorpipe.gor.model.DbConnection;
 import org.gorpipe.gor.session.ProjectContext;
@@ -63,9 +65,6 @@ public class QueryCommand extends HelpOptions implements Runnable{
     @CommandLine.Option(names={"-d","--cachedir"}, description = "Path to cache directory for the current gor query.")
     private Path cacheDir = Paths.get(ProjectContext.DEFAULT_CACHE_DIR);
 
-    @CommandLine.Option(defaultValue = "", names={"-p","--projectroot"}, description = "Sets the project root for the current gor query.")
-    private Path projectRoot;
-
     @CommandLine.Option(names={"-r","--requestid"}, description = "Sets a request id for the current gor query, used to identify logs and errors.")
     private String requestId = UUID.randomUUID().toString();
 
@@ -81,6 +80,9 @@ public class QueryCommand extends HelpOptions implements Runnable{
     @CommandLine.Parameters(index = "0", arity = "1", paramLabel = "InputQuery", description = "Queries to execute. Queries can be direct gor query, files containing gor script or gor report template.")
     private String query;
 
+    @CommandLine.ParentCommand
+    private GorCLI parentCommand;
+
     @Override
     public void run() {
 
@@ -95,8 +97,8 @@ public void run() {
             commandlineOptions.aliasFile_$eq(aliasFile.toString());
         if (configFile != null)
             commandlineOptions.configFile_$eq(configFile.toString());
-        if (projectRoot != null)
-            commandlineOptions.gorRoot_$eq(projectRoot.toString());
+        if (parentCommand.getProjectRoot() != null)
+            commandlineOptions.gorRoot_$eq(parentCommand.getProjectRoot().toString());
         commandlineOptions.requestId_$eq(requestId);
         commandlineOptions.showStackTrace_$eq(showStackTrace);
         commandlineOptions.workers_$eq(workers);
@@ -117,7 +119,9 @@ public void run() {
             // Initialize database connections
             DbConnection.initInConsoleApp();
 
-            GorExecutionEngine executionEngine = new CLIGorExecutionEngine(commandlineOptions, null, null);
+            var securityContext = System.getProperty("gor.security.context");
+
+            GorExecutionEngine executionEngine = new CLIGorExecutionEngine(commandlineOptions, null, securityContext);
             executionEngine.execute();
         } catch (GorException ge) {
             consoleLogger.error(ExceptionUtilities.gorExceptionToString(ge));

gorscripts/src/test/java/org/gorpipe/gor/cli/UTestGorCliManager.java

@@ -228,10 +228,10 @@ public void testFlagTagskeyValueCLI() throws Exception {
         Path dictFile = workDirPath.resolve(name + ".gord");
 
         //Insert 4 files with tags
-        GorCLI.main(new String[]{"manager", "insert", "--alias", "A", dictFile.toString(), testFiles[0]});
-        GorCLI.main(new String[]{"manager", "insert", "--alias", "B", dictFile.toString(), testFiles[1]});
-        GorCLI.main(new String[]{"manager", "insert", "--alias", "C", dictFile.toString(), testFiles[2]});
-        GorCLI.main(new String[]{"manager", "insert", "--alias", "D", dictFile.toString(), testFiles[3]});
+        GorExecCLI.main(new String[]{"manager", "insert", "--alias", "A", dictFile.toString(), testFiles[0]});
+        GorExecCLI.main(new String[]{"manager", "insert", "--alias", "B", dictFile.toString(), testFiles[1]});
+        GorExecCLI.main(new String[]{"manager", "insert", "--alias", "C", dictFile.toString(), testFiles[2]});
+        GorExecCLI.main(new String[]{"manager", "insert", "--alias", "D", dictFile.toString(), testFiles[3]});
 
         TableManager man = new TableManager();
         DictionaryTable<GorDictionaryEntry> table = man.initTable(dictFile);
@@ -242,14 +242,14 @@ public void testFlagTagskeyValueCLI() throws Exception {
 
         //Insert file 1 again but tag it the same as file 4 using the letter "D".
         //Using the --tagskey flag prevents the same tag to be used twice but instead the new line overwrites the old one.
-        GorCLI.main(new String[]{"manager", "insert", "--alias", "D", "--tagskey", dictFile.toString(), testFiles[0]});
+        GorExecCLI.main(new String[]{"manager", "insert", "--alias", "D", "--tagskey", dictFile.toString(), testFiles[0]});
         table.reload();
         result = table.selectUninon(table.filter()).stream().map(l -> l.formatEntry()).sorted().collect(Collectors.joining());
         Assert.assertEquals("Insert failed", testFiles[0] + "\tA\n" + testFiles[0] + "\tD\n" + testFiles[1] + "\tB\n" + testFiles[2] + "\tC\n", result);
 
         //Insert file 1 once again and tag it the same as file 2 using the letter "B"
         //This should be successful since the --tagskey option is not being used. There should now be two files tagged with "B".
-        GorCLI.main(new String[]{"manager", "insert", "--alias", "B", dictFile.toString(), testFiles[0]});
+        GorExecCLI.main(new String[]{"manager", "insert", "--alias", "B", dictFile.toString(), testFiles[0]});
         table.reload();
         //Total count of tags
         Assert.assertEquals(5, table.filter().get().stream().map(l -> l.getFilterTags()).distinct().count());
@@ -266,9 +266,9 @@ public void testFlagTagskeyValueCLI() throws Exception {
         }
 
         //Delete "B" so now there is only one remaining line with the same tag and it should be possible again to update the single existing line.
-        GorCLI.main(new String[]{"manager", "delete", "--aliases", "B", dictFile.toString(), testFiles[1]});
+        GorExecCLI.main(new String[]{"manager", "delete", "--aliases", "B", dictFile.toString(), testFiles[1]});
         //Insert tag "B" again which should update the current "B" tagged line with this entry.
-        GorCLI.main(new String[]{"manager", "insert", "--alias", "B", "--tagskey", dictFile.toString(), testFiles[3]});
+        GorExecCLI.main(new String[]{"manager", "insert", "--alias", "B", "--tagskey", dictFile.toString(), testFiles[3]});
         table.reload();
         result = table.selectUninon(table.filter()).stream().map(l -> l.formatEntry()).sorted().collect(Collectors.joining());
         Assert.assertEquals("Insert failed", testFiles[0] + "\tA\n" + testFiles[0] + "\tD\n" + testFiles[2] + "\tC\n" + testFiles[3] + "\tB\n", result);
@@ -283,21 +283,21 @@ public void testFlagTagskeyListCLI() throws Exception {
         DictionaryTable<GorDictionaryEntry> table = man.initTable(dictFile);
 
         //Check matching list of tags. If the same set of tags can be found it is replaced by the new line of tags.
-        GorCLI.main(new String[]{"manager", "insert", "--alias", "A", "--tags", "GO,RC,OR", "--tagskey", dictFile.toString(), testFiles[0]});
-        GorCLI.main(new String[]{"manager", "insert", "--alias", "A", "--tags", "GO,RC,OR", "--tagskey", dictFile.toString(), testFiles[1]});
+        GorExecCLI.main(new String[]{"manager", "insert", "--alias", "A", "--tags", "GO,RC,OR", "--tagskey", dictFile.toString(), testFiles[0]});
+        GorExecCLI.main(new String[]{"manager", "insert", "--alias", "A", "--tags", "GO,RC,OR", "--tagskey", dictFile.toString(), testFiles[1]});
         table.reload();
         String result = table.selectUninon(table.filter()).stream().map(l -> l.formatEntry()).sorted().collect(Collectors.joining());
         Assert.assertEquals("Insert failed", testFiles[1] + "\t\t\t\t\t\tGO,RC,OR,A\n", result);
 
         //Check a single tag against a list of tags to confirm that it is not enough to match a single tag in the list but the whole list must match.
-        GorCLI.main(new String[]{"manager", "insert", "--tags", "GO", "--tagskey", dictFile.toString(), testFiles[1]});
+        GorExecCLI.main(new String[]{"manager", "insert", "--tags", "GO", "--tagskey", dictFile.toString(), testFiles[1]});
         table.reload();
         result = table.selectUninon(table.filter()).stream().map(l -> l.formatEntry()).sorted().collect(Collectors.joining());
         Assert.assertEquals("Insert failed", testFiles[1] + "\t\t\t\t\t\tGO,RC,OR,A\n" + testFiles[1] + "\tGO\n", result);
 
         //Confirm the other flags such as range are ignored with the --tagskey flag. The second line "updates" the first line with range being ignored and tags only being used as an identifier.
-        GorCLI.main(new String[]{"manager", "insert", "--alias", "B", "--tags", "GO,RC,OR", "--tagskey", "--range","chr1-chr3", dictFile.toString(), testFiles[2]});
-        GorCLI.main(new String[]{"manager", "insert", "--alias", "B", "--tags", "GO,RC,OR", "--tagskey", dictFile.toString(), testFiles[3]});
+        GorExecCLI.main(new String[]{"manager", "insert", "--alias", "B", "--tags", "GO,RC,OR", "--tagskey", "--range","chr1-chr3", dictFile.toString(), testFiles[2]});
+        GorExecCLI.main(new String[]{"manager", "insert", "--alias", "B", "--tags", "GO,RC,OR", "--tagskey", dictFile.toString(), testFiles[3]});
         result = table.selectUninon(table.filter()).stream().map(l -> l.formatEntry()).sorted().collect(Collectors.joining());
         table.reload();
         Assert.assertEquals("Insert failed", testFiles[1] + "\t\t\t\t\t\tGO,RC,OR,A\n" + testFiles[1] + "\tGO\n" + testFiles[3] + "\t\t\t\t\t\tGO,RC,OR,B\n", result);
@@ -308,22 +308,22 @@ public void testDirectCLI() throws Exception {
         String name = "testDirectCLI";
         Path dictFile = workDirPath.resolve(name + ".gord");
 
-        GorCLI.main(new String[]{"manager", "insert", "--alias", "A", dictFile.toString(), testFiles[0]});
-        GorCLI.main(new String[]{"manager", "insert", "--alias", "B", dictFile.toString(), testFiles[1]});
-        GorCLI.main(new String[]{"manager", "insert", "--alias", "D", dictFile.toString(), testFiles[3]});
+        GorExecCLI.main(new String[]{"manager", "insert", "--alias", "A", dictFile.toString(), testFiles[0]});
+        GorExecCLI.main(new String[]{"manager", "insert", "--alias", "B", dictFile.toString(), testFiles[1]});
+        GorExecCLI.main(new String[]{"manager", "insert", "--alias", "D", dictFile.toString(), testFiles[3]});
 
         TableManager man = new TableManager();
         DictionaryTable<GorDictionaryEntry> table = man.initTable(dictFile);
 
         String result = table.selectUninon(table.filter()).stream().map(l -> l.formatEntry()).sorted().collect(Collectors.joining());
         Assert.assertEquals("Insert failed", testFiles[0] + "\tA\n" + testFiles[1] + "\tB\n" + testFiles[3] + "\tD\n", result);
 
-        GorCLI.main(new String[]{"manager", "delete", "--tags", "B", dictFile.toString()});
+        GorExecCLI.main(new String[]{"manager", "delete", "--tags", "B", dictFile.toString()});
         table.reload();
         result = table.selectUninon(table.filter().tags("B")).stream().map(l -> l.formatEntry()).sorted().collect(Collectors.joining());
         Assert.assertEquals("Delete failed", "", result);
 
-        GorCLI.main(new String[]{"manager", "bucketize", "-w", "1", "--min_bucket_size", "1", dictFile.toString()});
+        GorExecCLI.main(new String[]{"manager", "bucketize", "-w", "1", "--min_bucket_size", "1", dictFile.toString()});
         table.reload();
         Assert.assertEquals("Not all lines bucketized", 0, table.needsBucketizing().size());
     }