diff --git a/drivers/src/main/java/org/gorpipe/s3/shared/S3SharedSourceProvider.java b/drivers/src/main/java/org/gorpipe/s3/shared/S3SharedSourceProvider.java index 76562ed57..3d917bd22 100644 --- a/drivers/src/main/java/org/gorpipe/s3/shared/S3SharedSourceProvider.java +++ b/drivers/src/main/java/org/gorpipe/s3/shared/S3SharedSourceProvider.java @@ -97,18 +97,19 @@ public S3SharedSource resolveDataSource(SourceReference sourceReference) S3SharedSource source = null; - Credentials sharedCreds = getS3DataCredentials(getService(), sourceReference.getSecurityContext()); + BundledCredentials bundledCreds = BundledCredentials.fromSecurityContext(sourceReference.securityContext); + Credentials sharedCreds = getS3DataCredentials(getService(), bundledCreds ); if (sharedCreds == null) { log.warn(String.format("No credentials found for %s. Returning emtpy source.", getService())); return source; } - String project = Path.of(sourceReference.getCommonRoot()).getFileName().toString(); - String bucket = sharedCreds.getLookupKey(); - String s3SecurityContext = createS3SecurityContext(sharedCreds); + String s3SecurityContext = createSecurityContext(createS3CredsFromShared(sharedCreds), bundledCreds); String relativePath = getRelativePath(sourceReference.getUrl()); + String project = Path.of(sourceReference.getCommonRoot()).getFileName().toString(); + String bucket = sharedCreds.getLookupKey(); SourceReference s3SourceReference = createS3SourceReference(sourceReference, project, bucket, s3SecurityContext); S3Client client = getClient(s3SecurityContext, bucket); @@ -135,8 +136,7 @@ protected String findSharedSourceLinkContent(S3SharedSource source) { } } - private Credentials getS3DataCredentials(String service, String securityContext) { - BundledCredentials bundledCreds = BundledCredentials.fromSecurityContext(securityContext); + private Credentials getS3DataCredentials(String service, BundledCredentials bundledCreds) { List credsList = bundledCreds.getCredentialsForService(service); return getHighestPriorityCredential(credsList, service); } @@ -159,14 +159,21 @@ private Credentials getHighestPriorityCredential(List credsList, St return bestMatch; } - private String createS3SecurityContext(Credentials sharedCreds) { - BundledCredentials bundledCredentials = new BundledCredentials.Builder().addCredentials( - new Credentials("s3", sharedCreds.getLookupKey(), sharedCreds.getOwnerType(), - sharedCreds.getOwnerId(), sharedCreds.expires(), sharedCreds.isUserDefault(), - (Map) sharedCreds.toMap().get("credential_attributes"))) - .build(); - String securityContext = bundledCredentials.addToSecurityContext(""); - return securityContext; + private String createSecurityContext(Credentials s3Creds, BundledCredentials bundledCreds) { + BundledCredentials.Builder builder = new BundledCredentials.Builder().addCredentials(s3Creds); + for (String services : bundledCreds.services()) { + for (Credentials cred : bundledCreds.getCredentialsForService(services)) { + builder.addCredentials(cred); + } + } + + return builder.build().addToSecurityContext(""); + } + + private Credentials createS3CredsFromShared(Credentials sharedCreds) { + return new Credentials("s3", sharedCreds.getLookupKey(), sharedCreds.getOwnerType(), + sharedCreds.getOwnerId(), sharedCreds.expires(), sharedCreds.isUserDefault(), + (Map) sharedCreds.toMap().get("credential_attributes")); } private SourceReference createS3SourceReference(SourceReference sourceReference, String project, String bucket, String securityContext) { diff --git a/gorscripts/build.gradle b/gorscripts/build.gradle index b0ef52b55..673aa4a3c 100644 --- a/gorscripts/build.gradle +++ b/gorscripts/build.gradle @@ -48,7 +48,6 @@ project(':gorscripts') { implementation "org.apache.commons:commons-lang3:_" implementation "de.tototec:de.tototec.cmdoption:_" implementation "com.fasterxml.jackson.core:jackson-databind:_" - implementation "org.reflections:reflections:_" implementation "com.google.guava:guava:_" runtimeOnly project(':documentation') diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/GorCLI.java b/gorscripts/src/main/java/org/gorpipe/gor/cli/GorCLI.java index a3e201f54..f99e3a971 100644 --- a/gorscripts/src/main/java/org/gorpipe/gor/cli/GorCLI.java +++ b/gorscripts/src/main/java/org/gorpipe/gor/cli/GorCLI.java @@ -38,10 +38,8 @@ @CommandLine.Command(name="gor", version="version 1.0", description = "Command line interface for gor query language and processes.", - subcommands = {QueryCommand.class, HelpCommand.class, ManagerCommand.class, IndexCommand.class, - CacheCommand.class, RenderCommand.class, InfoCommand.class, FolderMigratorCommand.class, - LinkCommand.class}) -public class GorCLI extends HelpOptions implements Runnable { + subcommands = {QueryCommand.class, FolderMigratorCommand.class}) +public class GorCLI extends GorExecCLI implements Runnable { public static void main(String[] args) { GorLogbackUtil.initLog("gor"); CommandLine cmd = new CommandLine(new GorCLI()); @@ -52,11 +50,6 @@ public static void main(String[] args) { } - @CommandLine.Option(names = {"-v", "--version"}, - versionHelp = true, - description = "Print version information and exits.") - boolean versionHelpRequested; - @Override public void run() { CommandLine.usage(this, System.err); diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/query/QueryCommand.java b/gorscripts/src/main/java/org/gorpipe/gor/cli/query/QueryCommand.java index 164ce9cc1..2c8554f6b 100644 --- a/gorscripts/src/main/java/org/gorpipe/gor/cli/query/QueryCommand.java +++ b/gorscripts/src/main/java/org/gorpipe/gor/cli/query/QueryCommand.java @@ -28,6 +28,8 @@ import gorsat.process.PipeOptions; import org.gorpipe.exceptions.ExceptionUtilities; import org.gorpipe.exceptions.GorException; +import org.gorpipe.gor.cli.GorCLI; +import org.gorpipe.gor.cli.GorExecCLI; import org.gorpipe.gor.cli.HelpOptions; import org.gorpipe.gor.model.DbConnection; import org.gorpipe.gor.session.ProjectContext; @@ -63,9 +65,6 @@ public class QueryCommand extends HelpOptions implements Runnable{ @CommandLine.Option(names={"-d","--cachedir"}, description = "Path to cache directory for the current gor query.") private Path cacheDir = Paths.get(ProjectContext.DEFAULT_CACHE_DIR); - @CommandLine.Option(defaultValue = "", names={"-p","--projectroot"}, description = "Sets the project root for the current gor query.") - private Path projectRoot; - @CommandLine.Option(names={"-r","--requestid"}, description = "Sets a request id for the current gor query, used to identify logs and errors.") private String requestId = UUID.randomUUID().toString(); @@ -81,6 +80,9 @@ public class QueryCommand extends HelpOptions implements Runnable{ @CommandLine.Parameters(index = "0", arity = "1", paramLabel = "InputQuery", description = "Queries to execute. Queries can be direct gor query, files containing gor script or gor report template.") private String query; + @CommandLine.ParentCommand + private GorCLI parentCommand; + @Override public void run() { @@ -95,8 +97,8 @@ public void run() { commandlineOptions.aliasFile_$eq(aliasFile.toString()); if (configFile != null) commandlineOptions.configFile_$eq(configFile.toString()); - if (projectRoot != null) - commandlineOptions.gorRoot_$eq(projectRoot.toString()); + if (parentCommand.getProjectRoot() != null) + commandlineOptions.gorRoot_$eq(parentCommand.getProjectRoot().toString()); commandlineOptions.requestId_$eq(requestId); commandlineOptions.showStackTrace_$eq(showStackTrace); commandlineOptions.workers_$eq(workers); @@ -117,7 +119,9 @@ public void run() { // Initialize database connections DbConnection.initInConsoleApp(); - GorExecutionEngine executionEngine = new CLIGorExecutionEngine(commandlineOptions, null, null); + var securityContext = System.getProperty("gor.security.context"); + + GorExecutionEngine executionEngine = new CLIGorExecutionEngine(commandlineOptions, null, securityContext); executionEngine.execute(); } catch (GorException ge) { consoleLogger.error(ExceptionUtilities.gorExceptionToString(ge)); diff --git a/gorscripts/src/test/java/org/gorpipe/gor/cli/UTestGorCliManager.java b/gorscripts/src/test/java/org/gorpipe/gor/cli/UTestGorCliManager.java index 9baf0e2aa..33e948953 100644 --- a/gorscripts/src/test/java/org/gorpipe/gor/cli/UTestGorCliManager.java +++ b/gorscripts/src/test/java/org/gorpipe/gor/cli/UTestGorCliManager.java @@ -228,10 +228,10 @@ public void testFlagTagskeyValueCLI() throws Exception { Path dictFile = workDirPath.resolve(name + ".gord"); //Insert 4 files with tags - GorCLI.main(new String[]{"manager", "insert", "--alias", "A", dictFile.toString(), testFiles[0]}); - GorCLI.main(new String[]{"manager", "insert", "--alias", "B", dictFile.toString(), testFiles[1]}); - GorCLI.main(new String[]{"manager", "insert", "--alias", "C", dictFile.toString(), testFiles[2]}); - GorCLI.main(new String[]{"manager", "insert", "--alias", "D", dictFile.toString(), testFiles[3]}); + GorExecCLI.main(new String[]{"manager", "insert", "--alias", "A", dictFile.toString(), testFiles[0]}); + GorExecCLI.main(new String[]{"manager", "insert", "--alias", "B", dictFile.toString(), testFiles[1]}); + GorExecCLI.main(new String[]{"manager", "insert", "--alias", "C", dictFile.toString(), testFiles[2]}); + GorExecCLI.main(new String[]{"manager", "insert", "--alias", "D", dictFile.toString(), testFiles[3]}); TableManager man = new TableManager(); DictionaryTable table = man.initTable(dictFile); @@ -242,14 +242,14 @@ public void testFlagTagskeyValueCLI() throws Exception { //Insert file 1 again but tag it the same as file 4 using the letter "D". //Using the --tagskey flag prevents the same tag to be used twice but instead the new line overwrites the old one. - GorCLI.main(new String[]{"manager", "insert", "--alias", "D", "--tagskey", dictFile.toString(), testFiles[0]}); + GorExecCLI.main(new String[]{"manager", "insert", "--alias", "D", "--tagskey", dictFile.toString(), testFiles[0]}); table.reload(); result = table.selectUninon(table.filter()).stream().map(l -> l.formatEntry()).sorted().collect(Collectors.joining()); Assert.assertEquals("Insert failed", testFiles[0] + "\tA\n" + testFiles[0] + "\tD\n" + testFiles[1] + "\tB\n" + testFiles[2] + "\tC\n", result); //Insert file 1 once again and tag it the same as file 2 using the letter "B" //This should be successful since the --tagskey option is not being used. There should now be two files tagged with "B". - GorCLI.main(new String[]{"manager", "insert", "--alias", "B", dictFile.toString(), testFiles[0]}); + GorExecCLI.main(new String[]{"manager", "insert", "--alias", "B", dictFile.toString(), testFiles[0]}); table.reload(); //Total count of tags Assert.assertEquals(5, table.filter().get().stream().map(l -> l.getFilterTags()).distinct().count()); @@ -266,9 +266,9 @@ public void testFlagTagskeyValueCLI() throws Exception { } //Delete "B" so now there is only one remaining line with the same tag and it should be possible again to update the single existing line. - GorCLI.main(new String[]{"manager", "delete", "--aliases", "B", dictFile.toString(), testFiles[1]}); + GorExecCLI.main(new String[]{"manager", "delete", "--aliases", "B", dictFile.toString(), testFiles[1]}); //Insert tag "B" again which should update the current "B" tagged line with this entry. - GorCLI.main(new String[]{"manager", "insert", "--alias", "B", "--tagskey", dictFile.toString(), testFiles[3]}); + GorExecCLI.main(new String[]{"manager", "insert", "--alias", "B", "--tagskey", dictFile.toString(), testFiles[3]}); table.reload(); result = table.selectUninon(table.filter()).stream().map(l -> l.formatEntry()).sorted().collect(Collectors.joining()); Assert.assertEquals("Insert failed", testFiles[0] + "\tA\n" + testFiles[0] + "\tD\n" + testFiles[2] + "\tC\n" + testFiles[3] + "\tB\n", result); @@ -283,21 +283,21 @@ public void testFlagTagskeyListCLI() throws Exception { DictionaryTable table = man.initTable(dictFile); //Check matching list of tags. If the same set of tags can be found it is replaced by the new line of tags. - GorCLI.main(new String[]{"manager", "insert", "--alias", "A", "--tags", "GO,RC,OR", "--tagskey", dictFile.toString(), testFiles[0]}); - GorCLI.main(new String[]{"manager", "insert", "--alias", "A", "--tags", "GO,RC,OR", "--tagskey", dictFile.toString(), testFiles[1]}); + GorExecCLI.main(new String[]{"manager", "insert", "--alias", "A", "--tags", "GO,RC,OR", "--tagskey", dictFile.toString(), testFiles[0]}); + GorExecCLI.main(new String[]{"manager", "insert", "--alias", "A", "--tags", "GO,RC,OR", "--tagskey", dictFile.toString(), testFiles[1]}); table.reload(); String result = table.selectUninon(table.filter()).stream().map(l -> l.formatEntry()).sorted().collect(Collectors.joining()); Assert.assertEquals("Insert failed", testFiles[1] + "\t\t\t\t\t\tGO,RC,OR,A\n", result); //Check a single tag against a list of tags to confirm that it is not enough to match a single tag in the list but the whole list must match. - GorCLI.main(new String[]{"manager", "insert", "--tags", "GO", "--tagskey", dictFile.toString(), testFiles[1]}); + GorExecCLI.main(new String[]{"manager", "insert", "--tags", "GO", "--tagskey", dictFile.toString(), testFiles[1]}); table.reload(); result = table.selectUninon(table.filter()).stream().map(l -> l.formatEntry()).sorted().collect(Collectors.joining()); Assert.assertEquals("Insert failed", testFiles[1] + "\t\t\t\t\t\tGO,RC,OR,A\n" + testFiles[1] + "\tGO\n", result); //Confirm the other flags such as range are ignored with the --tagskey flag. The second line "updates" the first line with range being ignored and tags only being used as an identifier. - GorCLI.main(new String[]{"manager", "insert", "--alias", "B", "--tags", "GO,RC,OR", "--tagskey", "--range","chr1-chr3", dictFile.toString(), testFiles[2]}); - GorCLI.main(new String[]{"manager", "insert", "--alias", "B", "--tags", "GO,RC,OR", "--tagskey", dictFile.toString(), testFiles[3]}); + GorExecCLI.main(new String[]{"manager", "insert", "--alias", "B", "--tags", "GO,RC,OR", "--tagskey", "--range","chr1-chr3", dictFile.toString(), testFiles[2]}); + GorExecCLI.main(new String[]{"manager", "insert", "--alias", "B", "--tags", "GO,RC,OR", "--tagskey", dictFile.toString(), testFiles[3]}); result = table.selectUninon(table.filter()).stream().map(l -> l.formatEntry()).sorted().collect(Collectors.joining()); table.reload(); Assert.assertEquals("Insert failed", testFiles[1] + "\t\t\t\t\t\tGO,RC,OR,A\n" + testFiles[1] + "\tGO\n" + testFiles[3] + "\t\t\t\t\t\tGO,RC,OR,B\n", result); @@ -308,9 +308,9 @@ public void testDirectCLI() throws Exception { String name = "testDirectCLI"; Path dictFile = workDirPath.resolve(name + ".gord"); - GorCLI.main(new String[]{"manager", "insert", "--alias", "A", dictFile.toString(), testFiles[0]}); - GorCLI.main(new String[]{"manager", "insert", "--alias", "B", dictFile.toString(), testFiles[1]}); - GorCLI.main(new String[]{"manager", "insert", "--alias", "D", dictFile.toString(), testFiles[3]}); + GorExecCLI.main(new String[]{"manager", "insert", "--alias", "A", dictFile.toString(), testFiles[0]}); + GorExecCLI.main(new String[]{"manager", "insert", "--alias", "B", dictFile.toString(), testFiles[1]}); + GorExecCLI.main(new String[]{"manager", "insert", "--alias", "D", dictFile.toString(), testFiles[3]}); TableManager man = new TableManager(); DictionaryTable table = man.initTable(dictFile); @@ -318,12 +318,12 @@ public void testDirectCLI() throws Exception { String result = table.selectUninon(table.filter()).stream().map(l -> l.formatEntry()).sorted().collect(Collectors.joining()); Assert.assertEquals("Insert failed", testFiles[0] + "\tA\n" + testFiles[1] + "\tB\n" + testFiles[3] + "\tD\n", result); - GorCLI.main(new String[]{"manager", "delete", "--tags", "B", dictFile.toString()}); + GorExecCLI.main(new String[]{"manager", "delete", "--tags", "B", dictFile.toString()}); table.reload(); result = table.selectUninon(table.filter().tags("B")).stream().map(l -> l.formatEntry()).sorted().collect(Collectors.joining()); Assert.assertEquals("Delete failed", "", result); - GorCLI.main(new String[]{"manager", "bucketize", "-w", "1", "--min_bucket_size", "1", dictFile.toString()}); + GorExecCLI.main(new String[]{"manager", "bucketize", "-w", "1", "--min_bucket_size", "1", dictFile.toString()}); table.reload(); Assert.assertEquals("Not all lines bucketized", 0, table.needsBucketizing().size()); } diff --git a/gorscripts/src/test/java/org/gorpipe/gor/cli/link/LinkCommandTest.java b/gorscripts/src/test/java/org/gorpipe/gor/cli/link/LinkCommandTest.java index d8a519fb6..226ea9af0 100644 --- a/gorscripts/src/test/java/org/gorpipe/gor/cli/link/LinkCommandTest.java +++ b/gorscripts/src/test/java/org/gorpipe/gor/cli/link/LinkCommandTest.java @@ -5,9 +5,11 @@ import java.nio.file.Path; import java.time.Instant; +import org.gorpipe.gor.cli.GorCLI; import org.gorpipe.gor.driver.linkfile.LinkFile; import org.gorpipe.gor.driver.providers.stream.sources.file.FileSource; import static org.junit.Assert.assertEquals; + import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -22,9 +24,9 @@ public class LinkCommandTest { @Test public void testUpdateCreatesLinkFileAndAppliesHeaders() throws Exception { Path linkFile = temp.getRoot().toPath().resolve("update_test.gor.link"); - CommandLine cmd = new CommandLine(new LinkCommand()); + CommandLine cmd = new CommandLine(new GorCLI()); - int exitCode = cmd.execute("update", linkFile.toString(), "data/file1.gor", "-h", "ENTRIES_COUNT_MAX=5"); + int exitCode = cmd.execute("link", "update", linkFile.toString(), "data/file1.gor", "-h", "ENTRIES_COUNT_MAX=5"); assertEquals(0, exitCode); LinkFile link = LinkFile.load(new FileSource(linkFile)); @@ -36,9 +38,9 @@ public void testUpdateCreatesLinkFileAndAppliesHeaders() throws Exception { @Test public void testUpdateWithMd5AndInfo() throws Exception { Path linkFile = temp.getRoot().toPath().resolve("update_md5_info.gor.link"); - CommandLine cmd = new CommandLine(new LinkCommand()); + CommandLine cmd = new CommandLine(new GorCLI()); - int exitCode = cmd.execute("update", linkFile.toString(), "data/file1.gor", + int exitCode = cmd.execute("link", "update", linkFile.toString(), "data/file1.gor", "-m", "abc123", "-i", "first entry"); assertEquals(0, exitCode); @@ -51,13 +53,13 @@ public void testUpdateWithMd5AndInfo() throws Exception { @Test public void testRollbackLatestEntry() throws Exception { Path linkFile = temp.getRoot().toPath().resolve("rollback_latest.gor.link"); - CommandLine cmd = new CommandLine(new LinkCommand()); + CommandLine cmd = new CommandLine(new GorCLI()); - cmd.execute("update", linkFile.toString(), "data/file1.gor"); + cmd.execute("link", "update", linkFile.toString(), "data/file1.gor"); Thread.sleep(5); - cmd.execute("update", linkFile.toString(), "data/file2.gor"); + cmd.execute("link", "update", linkFile.toString(), "data/file2.gor"); - int exitCode = cmd.execute("rollback", linkFile.toString()); + int exitCode = cmd.execute("link", "rollback", linkFile.toString()); assertEquals(0, exitCode); LinkFile link = LinkFile.load(new FileSource(linkFile)); @@ -68,17 +70,17 @@ public void testRollbackLatestEntry() throws Exception { @Test public void testRollbackToDate() throws Exception { Path linkFile = temp.getRoot().toPath().resolve("rollback_date.gor.link"); - CommandLine cmd = new CommandLine(new LinkCommand()); + CommandLine cmd = new CommandLine(new GorCLI()); - cmd.execute("update", linkFile.toString(), "data/file1.gor"); + cmd.execute("link", "update", linkFile.toString(), "data/file1.gor"); LinkFile first = LinkFile.load(new FileSource(linkFile)); long firstTimestamp = first.getLatestEntry().timestamp(); Thread.sleep(5); - cmd.execute("update", linkFile.toString(), "data/file2.gor"); + cmd.execute("link", "update", linkFile.toString(), "data/file2.gor"); String rollbackIso = Instant.ofEpochMilli(firstTimestamp).toString(); - int exitCode = cmd.execute("rollback", linkFile.toString(), "-d", rollbackIso); + int exitCode = cmd.execute("link", "rollback", linkFile.toString(), "-d", rollbackIso); assertEquals(0, exitCode); LinkFile link = LinkFile.load(new FileSource(linkFile)); @@ -89,27 +91,27 @@ public void testRollbackToDate() throws Exception { @Test public void testResolveLatestEntry() throws Exception { Path linkFile = temp.getRoot().toPath().resolve("resolve_latest.gor.link"); - CommandLine cmd = new CommandLine(new LinkCommand()); + CommandLine cmd = new CommandLine(new GorCLI()); - cmd.execute("update", linkFile.toString(), "data/file1.gor"); + cmd.execute("link", "update", linkFile.toString(), "data/file1.gor"); Thread.sleep(5); - cmd.execute("update", linkFile.toString(), "data/file2.gor"); + cmd.execute("link", "update", linkFile.toString(), "data/file2.gor"); - String resolved = executeAndCapture(cmd, "resolve", linkFile.toString()); + String resolved = executeAndCapture(cmd, "link", "resolve", linkFile.toString()); assertEquals(resolve(linkFile, "data/file2.gor"), resolved); } @Test public void testResolveSpecificDate() throws Exception { Path linkFile = temp.getRoot().toPath().resolve("resolve_date.gor.link"); - CommandLine cmd = new CommandLine(new LinkCommand()); + CommandLine cmd = new CommandLine(new GorCLI()); - cmd.execute("update", linkFile.toString(), "data/file1.gor"); + cmd.execute("link", "update", linkFile.toString(), "data/file1.gor"); long firstTimestamp = LinkFile.load(new FileSource(linkFile)).getLatestEntry().timestamp(); Thread.sleep(5); - cmd.execute("update", linkFile.toString(), "data/file2.gor"); + cmd.execute("link", "update", linkFile.toString(), "data/file2.gor"); - String resolved = executeAndCapture(cmd, "resolve", linkFile.toString(), + String resolved = executeAndCapture(cmd, "link", "resolve", linkFile.toString(), "-d", Instant.ofEpochMilli(firstTimestamp).toString()); assertEquals(resolve(linkFile, "data/file1.gor"), resolved); } @@ -117,15 +119,15 @@ public void testResolveSpecificDate() throws Exception { @Test public void testResolveFullEntry() throws Exception { Path linkFile = temp.getRoot().toPath().resolve("resolve_full.gor.link"); - CommandLine cmd = new CommandLine(new LinkCommand()); + CommandLine cmd = new CommandLine(new GorCLI()); - cmd.execute("update", linkFile.toString(), "data/file1.gor"); + cmd.execute("link", "update", linkFile.toString(), "data/file1.gor"); Thread.sleep(5); - cmd.execute("update", linkFile.toString(), "data/file2.gor"); + cmd.execute("link", "update", linkFile.toString(), "data/file2.gor"); String expectedEntry = LinkFile.load(new FileSource(linkFile)).getLatestEntry().format(); - String resolved = executeAndCapture(cmd, "resolve", linkFile.toString(), "-f"); - assertEquals(expectedEntry, resolved); + String resolved = executeAndCapture(cmd, "link", "resolve", linkFile.toString(), "-f"); + assertEquals(expectedEntry.replace('\t', ' '), resolved); } private String resolve(Path linkFile, String relative) { diff --git a/gortools/build.gradle b/gortools/build.gradle index b24375b6b..a8bb4c0dd 100644 --- a/gortools/build.gradle +++ b/gortools/build.gradle @@ -58,6 +58,9 @@ project(':gortools') { implementation 'org.apache.parquet:parquet-column:_' implementation 'org.apache.parquet:parquet-hadoop:_' implementation ('org.apache.hadoop:hadoop-common:_') + implementation "info.picocli:picocli:_" + implementation "info.picocli:picocli-shell-jline3:_" + implementation "org.reflections:reflections:_" runtimeOnly project(':drivers') diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/FormattingOptions.java b/gortools/src/main/java/org/gorpipe/gor/cli/FormattingOptions.java similarity index 100% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/FormattingOptions.java rename to gortools/src/main/java/org/gorpipe/gor/cli/FormattingOptions.java diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/GorConfigDoc.java b/gortools/src/main/java/org/gorpipe/gor/cli/GorConfigDoc.java similarity index 100% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/GorConfigDoc.java rename to gortools/src/main/java/org/gorpipe/gor/cli/GorConfigDoc.java diff --git a/gortools/src/main/java/org/gorpipe/gor/cli/GorExecCLI.java b/gortools/src/main/java/org/gorpipe/gor/cli/GorExecCLI.java new file mode 100644 index 000000000..a5fb0f8df --- /dev/null +++ b/gortools/src/main/java/org/gorpipe/gor/cli/GorExecCLI.java @@ -0,0 +1,79 @@ +/* + * BEGIN_COPYRIGHT + * + * Copyright (C) 2011-2013 deCODE genetics Inc. + * Copyright (C) 2013-2019 WuXi NextCode Inc. + * All Rights Reserved. + * + * GORpipe is free software: you can redistribute it and/or modify + * it under the terms of the AFFERO GNU General Public License as published by + * the Free Software Foundation. + * + * GORpipe is distributed "AS-IS" AND WITHOUT ANY WARRANTY OF ANY KIND, + * INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, + * NON-INFRINGEMENT, OR FITNESS FOR A PARTICULAR PURPOSE. See + * the AFFERO GNU General Public License for the complete license terms. + * + * You should have received a copy of the AFFERO GNU General Public License + * along with GORpipe. If not, see + * + * END_COPYRIGHT + */ + +package org.gorpipe.gor.cli; + +import org.gorpipe.gor.cli.cache.CacheCommand; +import org.gorpipe.gor.cli.git.GitCommand; +import org.gorpipe.gor.cli.help.HelpCommand; +import org.gorpipe.gor.cli.index.IndexCommand; +import org.gorpipe.gor.cli.info.InfoCommand; +import org.gorpipe.gor.cli.link.LinkCommand; +import org.gorpipe.gor.cli.manager.ManagerCommand; +import org.gorpipe.gor.cli.render.RenderCommand; +import org.gorpipe.logging.GorLogbackUtil; +import picocli.CommandLine; + +import java.nio.file.Path; + +@SuppressWarnings("squid:S106") +@CommandLine.Command(name="gor", + version="version 1.0", + description = "Command line interface for gor query language and processes.", + subcommands = {HelpCommand.class, ManagerCommand.class, IndexCommand.class, + CacheCommand.class, RenderCommand.class, InfoCommand.class, + LinkCommand.class, GitCommand.class}) +public class GorExecCLI extends HelpOptions implements Runnable { + public static void main(String[] args) { + GorLogbackUtil.initLog("gor"); + CommandLine cmd = new CommandLine(new GorExecCLI()); + cmd.parseWithHandlers( + new CommandLine.RunLast(), + CommandLine.defaultExceptionHandler().andExit(-1), + args); + + } + + @CommandLine.Option(names = {"-v", "--version"}, + versionHelp = true, + description = "Print version information and exits.") + boolean versionHelpRequested; + + @CommandLine.Option(defaultValue = "", names={"-p","--projectRoot"}, description = "Sets the project root for the current gor session.") + private Path projectRoot; + + @CommandLine.Option(defaultValue = "", names={"--securityContext"}, description = "Sets the security context for the current gor session.") + private String securityContext; + + @Override + public void run() { + CommandLine.usage(this, System.err); + } + + public String getProjectRoot() { + return projectRoot.toString(); + } + + public String getSecurityContext() { + return securityContext; + } +} diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/HelpOptions.java b/gortools/src/main/java/org/gorpipe/gor/cli/HelpOptions.java similarity index 95% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/HelpOptions.java rename to gortools/src/main/java/org/gorpipe/gor/cli/HelpOptions.java index 775cbc48a..8fdc7b4a1 100644 --- a/gorscripts/src/main/java/org/gorpipe/gor/cli/HelpOptions.java +++ b/gortools/src/main/java/org/gorpipe/gor/cli/HelpOptions.java @@ -22,6 +22,7 @@ package org.gorpipe.gor.cli; +import org.gorpipe.gor.cli.FormattingOptions; import picocli.CommandLine; public class HelpOptions extends FormattingOptions { diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/cache/AnalysisResult.java b/gortools/src/main/java/org/gorpipe/gor/cli/cache/AnalysisResult.java similarity index 100% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/cache/AnalysisResult.java rename to gortools/src/main/java/org/gorpipe/gor/cli/cache/AnalysisResult.java diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/cache/CacheCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/cache/CacheCommand.java similarity index 100% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/cache/CacheCommand.java rename to gortools/src/main/java/org/gorpipe/gor/cli/cache/CacheCommand.java diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/cache/FilterOptions.java b/gortools/src/main/java/org/gorpipe/gor/cli/cache/FilterOptions.java similarity index 100% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/cache/FilterOptions.java rename to gortools/src/main/java/org/gorpipe/gor/cli/cache/FilterOptions.java diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/cache/PurgeCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/cache/PurgeCommand.java similarity index 100% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/cache/PurgeCommand.java rename to gortools/src/main/java/org/gorpipe/gor/cli/cache/PurgeCommand.java diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/cache/TouchCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/cache/TouchCommand.java similarity index 100% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/cache/TouchCommand.java rename to gortools/src/main/java/org/gorpipe/gor/cli/cache/TouchCommand.java diff --git a/gortools/src/main/java/org/gorpipe/gor/cli/git/GitAddCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/git/GitAddCommand.java new file mode 100644 index 000000000..58fddd85d --- /dev/null +++ b/gortools/src/main/java/org/gorpipe/gor/cli/git/GitAddCommand.java @@ -0,0 +1,88 @@ +package org.gorpipe.gor.cli.git; + +import picocli.CommandLine; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; + +@CommandLine.Command(name = "add", + description = "Add file contents to the staging area.") +public class GitAddCommand implements Runnable { + + @CommandLine.Parameters(arity = "0..*", paramLabel = "FILE", + description = "Files to add to the staging area. Patterns can be used, e.g., '*.java'.") + private List files; + + @CommandLine.Option(names = {"-A", "--all"}, + description = "Stage all changes in the working tree.") + private boolean all; + + @CommandLine.Option(names = {"-u", "--update"}, + description = "Stage only tracked files that have been modified or deleted.") + private boolean update; + + @CommandLine.Option(names = {"-f", "--force"}, + description = "Allow adding otherwise ignored files.") + private boolean force; + + @CommandLine.Option(names = {"-v", "--verbose"}, + description = "Be verbose.") + private boolean verbose; + + @CommandLine.Option(names = {"-n", "--dry-run"}, + description = "Don't actually add the file(s), just show if they exist and/or will be ignored.") + private boolean dryRun; + + @CommandLine.Option(names = {"-p", "--patch"}, + description = "Interactively choose hunks of patch between the index and the work tree.") + private boolean patch; + + @CommandLine.Option(names = {"-d", "--directory"}, paramLabel = "DIRECTORY", + description = "Directory to work in.", defaultValue = ".") + private String directory; + + @CommandLine.ParentCommand + private GitCommand parentCommand; + + @CommandLine.Spec + private CommandLine.Model.CommandSpec spec; + + @Override + public void run() { + List args = new ArrayList<>(); + + if (all) { + args.add("-A"); + } + + if (update) { + args.add("-u"); + } + + if (force) { + args.add("-f"); + } + + if (verbose) { + args.add("-v"); + } + + if (dryRun) { + args.add("-n"); + } + + if (patch) { + args.add("-p"); + } + + if (files != null && !files.isEmpty()) { + args.addAll(files); + } + + File workingDir = parentCommand.getWorkingDirectory(directory); + + GitCommandExecutor.executeGitCommand("add", args, workingDir, spec); + } +} + diff --git a/gortools/src/main/java/org/gorpipe/gor/cli/git/GitCheckoutCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/git/GitCheckoutCommand.java new file mode 100644 index 000000000..32a5accf4 --- /dev/null +++ b/gortools/src/main/java/org/gorpipe/gor/cli/git/GitCheckoutCommand.java @@ -0,0 +1,77 @@ +package org.gorpipe.gor.cli.git; + +import picocli.CommandLine; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; + +@CommandLine.Command(name = "checkout", + description = "Switch branches or restore working tree files.") +public class GitCheckoutCommand implements Runnable { + + @CommandLine.Parameters(index = "0", arity = "1..*", paramLabel = "BRANCH_OR_FILE", + description = "Branch, commit or files to checkout.") + private List branchOrFiles; + + @CommandLine.Option(names = {"-d", "--directory"}, paramLabel = "DIRECTORY", + description = "Directory to work in.", defaultValue = ".") + private String directory; + + @CommandLine.Option(names = {"-b", "--branch"}, paramLabel = "NEW_BRANCH", + description = "Create a new branch and switch to it.") + private String newBranch; + + @CommandLine.Option(names = {"-B", "--force-branch"}, paramLabel = "NEW_BRANCH", + description = "Create a new branch or reset an existing branch and switch to it.") + private String forceBranch; + + @CommandLine.Option(names = {"-f", "--force"}, + description = "Force checkout (throw away local changes).") + private boolean force; + + @CommandLine.Option(names = {"-q", "--quiet"}, + description = "Suppress feedback messages.") + private boolean quiet; + + @CommandLine.ParentCommand + private GitCommand parentCommand; + + @CommandLine.Spec + private CommandLine.Model.CommandSpec spec; + + @Override + public void run() { + if (branchOrFiles == null || branchOrFiles.isEmpty()) { + throw new CommandLine.ParameterException(spec.commandLine(), + "At least one branch or file must be specified."); + } + + List args = new ArrayList<>(); + + if (newBranch != null) { + args.add("-b"); + args.add(newBranch); + } + + if (forceBranch != null) { + args.add("-B"); + args.add(forceBranch); + } + + if (force) { + args.add("-f"); + } + + if (quiet) { + args.add("-q"); + } + + args.addAll(branchOrFiles); + + File workingDir = parentCommand.getWorkingDirectory(directory); + + GitCommandExecutor.executeGitCommand("checkout", args, workingDir, spec); + } +} + diff --git a/gortools/src/main/java/org/gorpipe/gor/cli/git/GitCloneCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/git/GitCloneCommand.java new file mode 100644 index 000000000..df3bf4ff1 --- /dev/null +++ b/gortools/src/main/java/org/gorpipe/gor/cli/git/GitCloneCommand.java @@ -0,0 +1,84 @@ +package org.gorpipe.gor.cli.git; + +import picocli.CommandLine; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; + +@CommandLine.Command(name = "clone", + description = "Clone a repository into a new directory.") +public class GitCloneCommand implements Runnable { + + @CommandLine.Parameters(index = "0", paramLabel = "REPOSITORY", + description = "The repository to clone from, just the repository name, e.g. ref-gregor.") + private String repository; + + @CommandLine.Parameters(index = "1", arity = "0..1", paramLabel = "DIRECTORY", + description = "The name of a new directory to clone into.") + private String directory; + + @CommandLine.Option(names = {"--depth"}, paramLabel = "DEPTH", + description = "Create a shallow clone with a history truncated to the specified number of commits.") + private Integer depth; + + @CommandLine.Option(names = {"-b", "--branch"}, paramLabel = "BRANCH", + description = "Clone a specific branch instead of the branch pointed to by HEAD.") + private String branch; + + @CommandLine.Option(names = {"--single-branch"}, + description = "Clone only the history leading to the tip of a single branch.") + private boolean singleBranch; + + @CommandLine.Option(names = {"--no-checkout"}, + description = "Do not checkout HEAD after cloning is complete.") + private boolean noCheckout; + + @CommandLine.Option(names = {"--bare"}, + description = "Make a bare Git repository.") + private boolean bare; + + @CommandLine.ParentCommand + private GitCommand parentCommand; + + @CommandLine.Spec + private CommandLine.Model.CommandSpec spec; + + @Override + public void run() { + List args = new ArrayList<>(); + + if (depth != null) { + args.add("--depth"); + args.add(depth.toString()); + } + + if (branch != null) { + args.add("-b"); + args.add(branch); + } + + if (singleBranch) { + args.add("--single-branch"); + } + + if (noCheckout) { + args.add("--no-checkout"); + } + + if (bare) { + args.add("--bare"); + } + + args.add(parentCommand.getFullRepositoryPath(repository)); + + if (directory != null) { + args.add(directory); + } + + File workingDir = parentCommand.getWorkingDirectory("."); + + GitCommandExecutor.executeGitCommand("clone", args, workingDir, spec); + } +} + diff --git a/gortools/src/main/java/org/gorpipe/gor/cli/git/GitCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/git/GitCommand.java new file mode 100644 index 000000000..008f1e5e8 --- /dev/null +++ b/gortools/src/main/java/org/gorpipe/gor/cli/git/GitCommand.java @@ -0,0 +1,61 @@ +package org.gorpipe.gor.cli.git; + +import org.gorpipe.gor.cli.GorExecCLI; +import org.gorpipe.gor.cli.HelpOptions; +import org.gorpipe.gor.table.util.PathUtils; +import org.gorpipe.util.Strings; +import picocli.CommandLine; + +import java.io.File; + +@SuppressWarnings("squid:S106") +@CommandLine.Command(name = "git", + description = "Wrapper for git commands (clone, checkout, pull, push, commit, add, config).", + header = "Git command wrapper.", + subcommands = {GitCloneCommand.class, GitCheckoutCommand.class, GitPullCommand.class, GitPushCommand.class, GitCommitCommand.class, GitAddCommand.class, GitConfigCommand.class}) +public class GitCommand extends HelpOptions implements Runnable { + + @CommandLine.ParentCommand + private GorExecCLI parentCommand; + + @Override + public void run() { + CommandLine.usage(this, System.err); + } + + public String getSecurityContext() { + return parentCommand != null ? parentCommand.getSecurityContext() : ""; + } + + public String getProjectRoot() { + return parentCommand != null ? parentCommand.getProjectRoot() : ""; + } + + public String getFullRepositoryPath(String repository) { + if (repository.equals("origin")) { + return repository; + } + String user = System.getenv("GOR_GIT_USER"); + String pass = System.getenv("GOR_GIT_TOKEN"); + if (user != null && pass != null) { + var userPass = "%s:%s@".formatted(user, pass); + return "https://%sgithub.com/GeneDx/%s.git".formatted(userPass, repository); + } else { + return "git@github.com:GeneDx/%s.git".formatted(repository); + } + } + + public File getWorkingDirectory(String directory) { + File workingDir = null; + if (!Strings.isNullOrEmpty(directory)) { + + if (!Strings.isNullOrEmpty(parentCommand.getProjectRoot())) { + workingDir = new File(PathUtils.resolve(parentCommand.getProjectRoot(), directory).toString()); + } else { + workingDir = new File(directory); + } + } + return workingDir; + } +} + diff --git a/gortools/src/main/java/org/gorpipe/gor/cli/git/GitCommandExecutor.java b/gortools/src/main/java/org/gorpipe/gor/cli/git/GitCommandExecutor.java new file mode 100644 index 000000000..4a8a1c041 --- /dev/null +++ b/gortools/src/main/java/org/gorpipe/gor/cli/git/GitCommandExecutor.java @@ -0,0 +1,79 @@ +package org.gorpipe.gor.cli.git; + +import picocli.CommandLine; + +import java.io.BufferedReader; +import java.io.File; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; + +/** + * Utility class for executing git commands. + */ +class GitCommandExecutor { + + /** + * Execute a git command with the given arguments. + * + * @param gitSubcommand the git subcommand (e.g., "clone", "checkout", "pull", "push") + * @param args additional arguments to pass to git + * @param workingDir the working directory for the command (null for current directory) + * @param commandSpec the CommandLine spec for error reporting + * @return the exit code of the git command + */ + static int executeGitCommand(String gitSubcommand, List args, File workingDir, + CommandLine.Model.CommandSpec commandSpec) { + List command = new ArrayList<>(); + command.add("git"); + command.add(gitSubcommand); + command.addAll(args); + + try { + ProcessBuilder pb = new ProcessBuilder(command); + if (workingDir != null) { + pb.directory(workingDir); + } + pb.redirectErrorStream(false); + + Process process = pb.start(); + + // Stream stdout to System.out + streamOutput(process.getInputStream(), System.out); + + // Stream stderr to System.err + streamOutput(process.getErrorStream(), System.err); + + int exitCode = process.waitFor(); + + if (exitCode != 0) { + throw new CommandLine.ExecutionException(commandSpec.commandLine(), + String.format("Git command '%s' failed with exit code %d", gitSubcommand, exitCode)); + } + + return exitCode; + } catch (CommandLine.ExecutionException e) { + throw e; + } catch (Exception e) { + throw new CommandLine.ExecutionException(commandSpec.commandLine(), + String.format("Failed to execute git command '%s': %s", gitSubcommand, e.getMessage()), e); + } + } + + private static void streamOutput(InputStream inputStream, java.io.PrintStream outputStream) { + new Thread(() -> { + try (BufferedReader reader = new BufferedReader( + new InputStreamReader(inputStream, StandardCharsets.UTF_8))) { + String line; + while ((line = reader.readLine()) != null) { + outputStream.println(line); + } + } catch (Exception e) { + // Ignore errors in output streaming + } + }).start(); + } +} + diff --git a/gortools/src/main/java/org/gorpipe/gor/cli/git/GitCommitCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/git/GitCommitCommand.java new file mode 100644 index 000000000..d255a3223 --- /dev/null +++ b/gortools/src/main/java/org/gorpipe/gor/cli/git/GitCommitCommand.java @@ -0,0 +1,89 @@ +package org.gorpipe.gor.cli.git; + +import picocli.CommandLine; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; + +@CommandLine.Command(name = "commit", + description = "Record changes to the repository.") +public class GitCommitCommand implements Runnable { + + @CommandLine.Option(names = {"-m", "--message"}, paramLabel = "MESSAGE", + description = "Use the given message as the commit message.") + private String message; + + @CommandLine.Option(names = {"-a", "--all"}, + description = "Automatically stage files that have been modified and deleted.") + private boolean all; + + @CommandLine.Option(names = {"--amend"}, + description = "Amend the previous commit.") + private boolean amend; + + @CommandLine.Option(names = {"--no-verify", "-n"}, + description = "Bypass pre-commit and commit-msg hooks.") + private boolean noVerify; + + @CommandLine.Option(names = {"-v", "--verbose"}, + description = "Show unified diff of all file changes in the commit message template.") + private boolean verbose; + + @CommandLine.Option(names = {"-q", "--quiet"}, + description = "Suppress commit summary message.") + private boolean quiet; + + @CommandLine.Option(names = {"-d", "--directory"}, paramLabel = "DIRECTORY", + description = "Directory to work in.", defaultValue = ".") + private String directory; + + @CommandLine.Parameters(arity = "0..*", paramLabel = "FILE", + description = "Files to commit. If not specified, all staged files are committed.") + private List files; + + @CommandLine.ParentCommand + private GitCommand parentCommand; + + @CommandLine.Spec + private CommandLine.Model.CommandSpec spec; + + @Override + public void run() { + List args = new ArrayList<>(); + + if (all) { + args.add("-a"); + } + + if (amend) { + args.add("--amend"); + } + + if (noVerify) { + args.add("--no-verify"); + } + + if (verbose) { + args.add("-v"); + } + + if (quiet) { + args.add("-q"); + } + + if (message != null) { + args.add("-m"); + args.add(message); + } + + if (files != null && !files.isEmpty()) { + args.addAll(files); + } + + File workingDir = parentCommand.getWorkingDirectory(directory); + + GitCommandExecutor.executeGitCommand("commit", args, workingDir, spec); + } +} + diff --git a/gortools/src/main/java/org/gorpipe/gor/cli/git/GitConfigCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/git/GitConfigCommand.java new file mode 100644 index 000000000..bbf919095 --- /dev/null +++ b/gortools/src/main/java/org/gorpipe/gor/cli/git/GitConfigCommand.java @@ -0,0 +1,104 @@ +package org.gorpipe.gor.cli.git; + +import picocli.CommandLine; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; + +@CommandLine.Command(name = "config", + description = "Get and set repository or global options.") +public class GitConfigCommand implements Runnable { + + @CommandLine.Parameters(arity = "0..*", paramLabel = "KEY [VALUE]", + description = "Configuration key, optionally followed by value to set.") + private List configArgs; + + @CommandLine.Option(names = {"--global"}, + description = "Use global config file.") + private boolean global; + + @CommandLine.Option(names = {"--local"}, + description = "Use repository config file (default).") + private boolean local; + + @CommandLine.Option(names = {"--system"}, + description = "Use system config file.") + private boolean system; + + @CommandLine.Option(names = {"--list", "-l"}, + description = "List all variables set in config file.") + private boolean list; + + @CommandLine.Option(names = {"--get"}, + description = "Get the value for a given key.") + private boolean get; + + @CommandLine.Option(names = {"--set"}, + description = "Set a value for a given key.") + private boolean set; + + @CommandLine.Option(names = {"--unset"}, + description = "Remove a key from the config file.") + private boolean unset; + + @CommandLine.Option(names = {"--unset-all"}, + description = "Remove all matches for a key from the config file.") + private boolean unsetAll; + + @CommandLine.Option(names = {"-d", "--directory"}, paramLabel = "DIRECTORY", + description = "Directory to work in.", defaultValue = ".") + private String directory; + + @CommandLine.ParentCommand + private GitCommand parentCommand; + + @CommandLine.Spec + private CommandLine.Model.CommandSpec spec; + + @Override + public void run() { + List args = new ArrayList<>(); + + if (global) { + args.add("--global"); + } + + if (local) { + args.add("--local"); + } + + if (system) { + args.add("--system"); + } + + if (list) { + args.add("--list"); + } + + if (get) { + args.add("--get"); + } + + if (set) { + args.add("--set"); + } + + if (unset) { + args.add("--unset"); + } + + if (unsetAll) { + args.add("--unset-all"); + } + + if (configArgs != null && !configArgs.isEmpty()) { + args.addAll(configArgs); + } + + File workingDir = parentCommand.getWorkingDirectory(directory); + + GitCommandExecutor.executeGitCommand("config", args, workingDir, spec); + } +} + diff --git a/gortools/src/main/java/org/gorpipe/gor/cli/git/GitPullCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/git/GitPullCommand.java new file mode 100644 index 000000000..255a9be00 --- /dev/null +++ b/gortools/src/main/java/org/gorpipe/gor/cli/git/GitPullCommand.java @@ -0,0 +1,96 @@ +package org.gorpipe.gor.cli.git; + +import picocli.CommandLine; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; + +@CommandLine.Command(name = "pull", + description = "Fetch from and integrate with another repository or a local branch.") +public class GitPullCommand implements Runnable { + + @CommandLine.Parameters(index = "0", arity = "0..1", paramLabel = "REPOSITORY", + description = "The repository to pull from.") + private String repository; + + @CommandLine.Option(names = {"-d", "--directory"}, paramLabel = "DIRECTORY", + description = "Directory to work in.", defaultValue = ".") + private String directory; + + @CommandLine.Parameters(index = "1", arity = "0..1", paramLabel = "REFSPEC", + description = "The branch or commit to pull.") + private String refspec; + + @CommandLine.Option(names = {"--rebase"}, + description = "Rebase the current branch on top of the upstream branch.") + private boolean rebase; + + @CommandLine.Option(names = {"--no-rebase"}, + description = "Merge the remote-tracking branch into the current branch (default).") + private boolean noRebase; + + @CommandLine.Option(names = {"--ff-only"}, + description = "Refuse to merge unless the current HEAD is already up to date or the merge can be resolved as a fast-forward.") + private boolean ffOnly; + + @CommandLine.Option(names = {"--no-ff"}, + description = "Create a merge commit even when the merge could be resolved as a fast-forward.") + private boolean noff; + + @CommandLine.Option(names = {"-q", "--quiet"}, + description = "Operate quietly.") + private boolean quiet; + + @CommandLine.Option(names = {"-v", "--verbose"}, + description = "Be verbose.") + private boolean verbose; + + @CommandLine.ParentCommand + private GitCommand parentCommand; + + @CommandLine.Spec + private CommandLine.Model.CommandSpec spec; + + @Override + public void run() { + List args = new ArrayList<>(); + + if (rebase) { + args.add("--rebase"); + } + + if (noRebase) { + args.add("--no-rebase"); + } + + if (ffOnly) { + args.add("--ff-only"); + } + + if (noff) { + args.add("--no-ff"); + } + + if (quiet) { + args.add("-q"); + } + + if (verbose) { + args.add("-v"); + } + + if (repository != null) { + args.add(parentCommand.getFullRepositoryPath(repository)); + } + + if (refspec != null) { + args.add(refspec); + } + + File workingDir = parentCommand.getWorkingDirectory(directory); + + GitCommandExecutor.executeGitCommand("pull", args, workingDir, spec); + } +} + diff --git a/gortools/src/main/java/org/gorpipe/gor/cli/git/GitPushCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/git/GitPushCommand.java new file mode 100644 index 000000000..fef0ee75b --- /dev/null +++ b/gortools/src/main/java/org/gorpipe/gor/cli/git/GitPushCommand.java @@ -0,0 +1,96 @@ +package org.gorpipe.gor.cli.git; + +import picocli.CommandLine; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; + +@CommandLine.Command(name = "push", + description = "Update remote refs along with associated objects.") +public class GitPushCommand implements Runnable { + + @CommandLine.Parameters(index = "0", arity = "0..1", paramLabel = "REPOSITORY", + description = "The repository to push to.") + private String repository; + + @CommandLine.Parameters(index = "1", arity = "0..1", paramLabel = "REFSPEC", + description = "The branch or ref to push.") + private String refspec; + + @CommandLine.Option(names = {"-d", "--directory"}, paramLabel = "DIRECTORY", + description = "Directory to work in.", defaultValue = ".") + private String directory; + + @CommandLine.Option(names = {"-u", "--set-upstream"}, + description = "Set upstream for git pull/status.") + private boolean setUpstream; + + @CommandLine.Option(names = {"--force", "-f"}, + description = "Force update remote refs.") + private boolean force; + + @CommandLine.Option(names = {"--force-with-lease"}, + description = "Force update, but refuse to update if the remote-tracking ref has been updated.") + private boolean forceWithLease; + + @CommandLine.Option(names = {"-q", "--quiet"}, + description = "Operate quietly.") + private boolean quiet; + + @CommandLine.Option(names = {"-v", "--verbose"}, + description = "Be verbose.") + private boolean verbose; + + @CommandLine.Option(names = {"--tags"}, + description = "Push all refs under refs/tags.") + private boolean tags; + + @CommandLine.ParentCommand + private GitCommand parentCommand; + + @CommandLine.Spec + private CommandLine.Model.CommandSpec spec; + + @Override + public void run() { + List args = new ArrayList<>(); + + if (setUpstream) { + args.add("-u"); + } + + if (force) { + args.add("--force"); + } + + if (forceWithLease) { + args.add("--force-with-lease"); + } + + if (quiet) { + args.add("-q"); + } + + if (verbose) { + args.add("-v"); + } + + if (tags) { + args.add("--tags"); + } + + if (repository != null) { + args.add(parentCommand.getFullRepositoryPath(repository)); + } + + if (refspec != null) { + args.add(refspec); + } + + File workingDir = parentCommand.getWorkingDirectory(directory); + + GitCommandExecutor.executeGitCommand("push", args, workingDir, spec); + } +} + diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/help/HelpCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/help/HelpCommand.java similarity index 97% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/help/HelpCommand.java rename to gortools/src/main/java/org/gorpipe/gor/cli/help/HelpCommand.java index 3ecf4ce7f..33a771c42 100644 --- a/gorscripts/src/main/java/org/gorpipe/gor/cli/help/HelpCommand.java +++ b/gortools/src/main/java/org/gorpipe/gor/cli/help/HelpCommand.java @@ -109,7 +109,7 @@ private List loadHelpFiles() throws IOException, URISyntaxException { Map env = new HashMap<>(); env.put("create", "true"); try(FileSystem fileSystem = java.nio.file.FileSystems.newFileSystem(helpJarURL, env)) { - helpList.addAll(java.nio.file.Files.readAllLines(java.nio.file.Paths.get(helpJarURL), java.nio.charset.Charset.forName("ISO-8859-1"))); + helpList.addAll(java.nio.file.Files.readAllLines(java.nio.file.Paths.get(helpJarURL), Charset.forName("ISO-8859-1"))); } } diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/index/IndexCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/index/IndexCommand.java similarity index 100% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/index/IndexCommand.java rename to gortools/src/main/java/org/gorpipe/gor/cli/index/IndexCommand.java diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/info/CommandsCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/info/CommandsCommand.java similarity index 100% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/info/CommandsCommand.java rename to gortools/src/main/java/org/gorpipe/gor/cli/info/CommandsCommand.java diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/info/InfoCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/info/InfoCommand.java similarity index 100% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/info/InfoCommand.java rename to gortools/src/main/java/org/gorpipe/gor/cli/info/InfoCommand.java diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/info/InputsCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/info/InputsCommand.java similarity index 100% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/info/InputsCommand.java rename to gortools/src/main/java/org/gorpipe/gor/cli/info/InputsCommand.java diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/info/MacrosCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/info/MacrosCommand.java similarity index 100% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/info/MacrosCommand.java rename to gortools/src/main/java/org/gorpipe/gor/cli/info/MacrosCommand.java diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/link/LinkCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/link/LinkCommand.java similarity index 64% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/link/LinkCommand.java rename to gortools/src/main/java/org/gorpipe/gor/cli/link/LinkCommand.java index c83be67cf..a4fb3fc5e 100644 --- a/gorscripts/src/main/java/org/gorpipe/gor/cli/link/LinkCommand.java +++ b/gortools/src/main/java/org/gorpipe/gor/cli/link/LinkCommand.java @@ -1,5 +1,6 @@ package org.gorpipe.gor.cli.link; +import org.gorpipe.gor.cli.GorExecCLI; import org.gorpipe.gor.cli.HelpOptions; import picocli.CommandLine; @@ -10,8 +11,19 @@ subcommands = {LinkUpdateCommand.class, LinkRollbackCommand.class, LinkResolveCommand.class}) public class LinkCommand extends HelpOptions implements Runnable { + @CommandLine.ParentCommand + private GorExecCLI parentCommand; + @Override public void run() { CommandLine.usage(this, System.err); } + + public String getSecurityContext() { + return parentCommand.getSecurityContext(); + } + + public String getProjectRoot() { + return parentCommand.getProjectRoot(); + } } diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/link/LinkResolveCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/link/LinkResolveCommand.java similarity index 91% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/link/LinkResolveCommand.java rename to gortools/src/main/java/org/gorpipe/gor/cli/link/LinkResolveCommand.java index 23a22671b..223d3300b 100644 --- a/gorscripts/src/main/java/org/gorpipe/gor/cli/link/LinkResolveCommand.java +++ b/gortools/src/main/java/org/gorpipe/gor/cli/link/LinkResolveCommand.java @@ -1,14 +1,13 @@ package org.gorpipe.gor.cli.link; -import java.io.IOException; -import java.time.Instant; - import org.gorpipe.gor.driver.linkfile.LinkFile; import org.gorpipe.util.DateUtils; import org.gorpipe.util.Strings; - import picocli.CommandLine; +import java.io.IOException; +import java.time.Instant; + @CommandLine.Command(name = "resolve", description = "Resolve a link file to the entry active at the current or given time.") public class LinkResolveCommand implements Runnable { @@ -25,11 +24,14 @@ public class LinkResolveCommand implements Runnable { description = "Return the full link file entry instead of only the resolved URL.") private boolean returnFullEntry; + @CommandLine.ParentCommand + private LinkCommand mainCmd; + @Override public void run() { var normalizedLinkPath = LinkFile.validateAndUpdateLinkFileName(linkFilePath); try { - var linkFile = LinkFile.load(LinkStreamSourceProvider.resolve(normalizedLinkPath, true, this)); + var linkFile = LinkFile.load(LinkStreamSourceProvider.resolve(normalizedLinkPath, mainCmd.getSecurityContext(), mainCmd.getProjectRoot(), true, this)); long timestamp = resolveDate == null ? System.currentTimeMillis() : parseDate(resolveDate); var entry = linkFile.getEntry(timestamp); if (entry == null) { @@ -38,7 +40,7 @@ public void run() { } String output; if (returnFullEntry) { - output = entry.format(); + output = entry.format().replace('\t', ' '); } else { var resolved = linkFile.getEntryUrl(timestamp); if (Strings.isNullOrEmpty(resolved)) { diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/link/LinkRollbackCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/link/LinkRollbackCommand.java similarity index 92% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/link/LinkRollbackCommand.java rename to gortools/src/main/java/org/gorpipe/gor/cli/link/LinkRollbackCommand.java index 4a239fb76..04f8c1769 100644 --- a/gorscripts/src/main/java/org/gorpipe/gor/cli/link/LinkRollbackCommand.java +++ b/gortools/src/main/java/org/gorpipe/gor/cli/link/LinkRollbackCommand.java @@ -1,13 +1,12 @@ package org.gorpipe.gor.cli.link; -import java.io.IOException; -import java.time.Instant; - import org.gorpipe.gor.driver.linkfile.LinkFile; import org.gorpipe.util.DateUtils; - import picocli.CommandLine; +import java.io.IOException; +import java.time.Instant; + @CommandLine.Command(name = "rollback", description = "Rollback the latest entry or rollback entries newer than a given date.") public class LinkRollbackCommand implements Runnable { @@ -19,11 +18,14 @@ public class LinkRollbackCommand implements Runnable { description = "ISO-8601 date/time or epoch milliseconds to rollback to (entries newer than this are removed).") private String rollbackDate; + @CommandLine.ParentCommand + private LinkCommand mainCmd; + @Override public void run() { var normalizedLinkPath = LinkFile.validateAndUpdateLinkFileName(linkFilePath); try { - var linkFile = LinkFile.load(LinkStreamSourceProvider.resolve(normalizedLinkPath, true, this)); + var linkFile = LinkFile.load(LinkStreamSourceProvider.resolve(normalizedLinkPath, mainCmd.getSecurityContext(), mainCmd.getProjectRoot(), true, this)); boolean changed = rollbackDate == null ? linkFile.rollbackLatestEntry() : linkFile.rollbackToTimestamp(parseDate(rollbackDate)); if (!changed) { throw new CommandLine.ParameterException(new CommandLine(this), diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/link/LinkStreamSourceProvider.java b/gortools/src/main/java/org/gorpipe/gor/cli/link/LinkStreamSourceProvider.java similarity index 78% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/link/LinkStreamSourceProvider.java rename to gortools/src/main/java/org/gorpipe/gor/cli/link/LinkStreamSourceProvider.java index bfde54a84..4687500f1 100644 --- a/gorscripts/src/main/java/org/gorpipe/gor/cli/link/LinkStreamSourceProvider.java +++ b/gortools/src/main/java/org/gorpipe/gor/cli/link/LinkStreamSourceProvider.java @@ -9,8 +9,8 @@ final class LinkStreamSourceProvider { private LinkStreamSourceProvider() { } - static StreamSource resolve(String linkPath, boolean writeable, Object commandInstance) { - var fileReader = new DriverBackedFileReader("", null); + static StreamSource resolve(String linkPath, String securityContext, String commonRoot, boolean writeable, Object commandInstance) { + var fileReader = new DriverBackedFileReader(securityContext, commonRoot); var dataSource = fileReader.resolveUrl(linkPath, writeable); if (dataSource == null) { throw new CommandLine.ExecutionException(new CommandLine(commandInstance), diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/link/LinkUpdateCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/link/LinkUpdateCommand.java similarity index 91% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/link/LinkUpdateCommand.java rename to gortools/src/main/java/org/gorpipe/gor/cli/link/LinkUpdateCommand.java index c6247e3d6..1875043be 100644 --- a/gorscripts/src/main/java/org/gorpipe/gor/cli/link/LinkUpdateCommand.java +++ b/gortools/src/main/java/org/gorpipe/gor/cli/link/LinkUpdateCommand.java @@ -1,14 +1,13 @@ package org.gorpipe.gor.cli.link; -import java.io.IOException; -import java.util.LinkedHashMap; -import java.util.Map; - import org.gorpipe.gor.driver.linkfile.LinkFile; import org.gorpipe.util.Strings; - import picocli.CommandLine; +import java.io.IOException; +import java.util.LinkedHashMap; +import java.util.Map; + @CommandLine.Command(name = "update", description = "Append a new entry to a link file, creating the file if needed.") public class LinkUpdateCommand implements Runnable { @@ -33,15 +32,18 @@ public class LinkUpdateCommand implements Runnable { mapFallbackValue = "") private final Map headerParams = new LinkedHashMap<>(); + @CommandLine.ParentCommand + private LinkCommand mainCmd; + @Override public void run() { var normalizedLinkPath = LinkFile.validateAndUpdateLinkFileName(linkFilePath); try { - var linkFile = LinkFile.load(LinkStreamSourceProvider.resolve(normalizedLinkPath, true, this)); + var linkFile = LinkFile.load(LinkStreamSourceProvider.resolve(normalizedLinkPath, mainCmd.getSecurityContext(), mainCmd.getProjectRoot(), true, this)); applyHeaders(linkFile); linkFile.appendEntry(linkValue, entryMd5, entryInfo); linkFile.save(); - System.out.printf("Updated link file %s with %s%n", normalizedLinkPath, linkValue); + System.err.printf("Updated link file %s with %s%n", normalizedLinkPath, linkValue); } catch (IOException e) { throw new CommandLine.ExecutionException(new CommandLine(this), "Failed to load or create link file: " + normalizedLinkPath, e); diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/manager/BucketizeCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/manager/BucketizeCommand.java similarity index 98% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/manager/BucketizeCommand.java rename to gortools/src/main/java/org/gorpipe/gor/cli/manager/BucketizeCommand.java index ec8d2f2c9..494fa7f94 100644 --- a/gorscripts/src/main/java/org/gorpipe/gor/cli/manager/BucketizeCommand.java +++ b/gortools/src/main/java/org/gorpipe/gor/cli/manager/BucketizeCommand.java @@ -26,8 +26,6 @@ import org.gorpipe.gor.manager.TableManager; import picocli.CommandLine; -import java.net.URI; -import java.nio.file.Paths; import java.time.Duration; import java.util.ArrayList; import java.util.List; diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/manager/CommandBucketizeOptions.java b/gortools/src/main/java/org/gorpipe/gor/cli/manager/CommandBucketizeOptions.java similarity index 100% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/manager/CommandBucketizeOptions.java rename to gortools/src/main/java/org/gorpipe/gor/cli/manager/CommandBucketizeOptions.java diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/manager/DeleteBucketCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/manager/DeleteBucketCommand.java similarity index 96% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/manager/DeleteBucketCommand.java rename to gortools/src/main/java/org/gorpipe/gor/cli/manager/DeleteBucketCommand.java index 9ac45be35..fbc589923 100644 --- a/gorscripts/src/main/java/org/gorpipe/gor/cli/manager/DeleteBucketCommand.java +++ b/gortools/src/main/java/org/gorpipe/gor/cli/manager/DeleteBucketCommand.java @@ -25,8 +25,6 @@ import org.gorpipe.gor.manager.TableManager; import picocli.CommandLine; -import java.nio.file.Path; -import java.nio.file.Paths; import java.time.Duration; import java.util.ArrayList; import java.util.List; diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/manager/DeleteCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/manager/DeleteCommand.java similarity index 100% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/manager/DeleteCommand.java rename to gortools/src/main/java/org/gorpipe/gor/cli/manager/DeleteCommand.java diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/manager/FilterOptions.java b/gortools/src/main/java/org/gorpipe/gor/cli/manager/FilterOptions.java similarity index 100% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/manager/FilterOptions.java rename to gortools/src/main/java/org/gorpipe/gor/cli/manager/FilterOptions.java diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/manager/InsertCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/manager/InsertCommand.java similarity index 100% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/manager/InsertCommand.java rename to gortools/src/main/java/org/gorpipe/gor/cli/manager/InsertCommand.java diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/manager/ManagerCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/manager/ManagerCommand.java similarity index 100% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/manager/ManagerCommand.java rename to gortools/src/main/java/org/gorpipe/gor/cli/manager/ManagerCommand.java diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/manager/ManagerOptions.java b/gortools/src/main/java/org/gorpipe/gor/cli/manager/ManagerOptions.java similarity index 100% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/manager/ManagerOptions.java rename to gortools/src/main/java/org/gorpipe/gor/cli/manager/ManagerOptions.java diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/manager/MultiInsertCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/manager/MultiInsertCommand.java similarity index 100% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/manager/MultiInsertCommand.java rename to gortools/src/main/java/org/gorpipe/gor/cli/manager/MultiInsertCommand.java diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/manager/SelectCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/manager/SelectCommand.java similarity index 100% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/manager/SelectCommand.java rename to gortools/src/main/java/org/gorpipe/gor/cli/manager/SelectCommand.java diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/manager/TestCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/manager/TestCommand.java similarity index 100% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/manager/TestCommand.java rename to gortools/src/main/java/org/gorpipe/gor/cli/manager/TestCommand.java diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/manager/TestIsLockCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/manager/TestIsLockCommand.java similarity index 100% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/manager/TestIsLockCommand.java rename to gortools/src/main/java/org/gorpipe/gor/cli/manager/TestIsLockCommand.java diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/manager/TestReadLockCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/manager/TestReadLockCommand.java similarity index 100% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/manager/TestReadLockCommand.java rename to gortools/src/main/java/org/gorpipe/gor/cli/manager/TestReadLockCommand.java diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/manager/TestWriteLockCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/manager/TestWriteLockCommand.java similarity index 100% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/manager/TestWriteLockCommand.java rename to gortools/src/main/java/org/gorpipe/gor/cli/manager/TestWriteLockCommand.java diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/render/RenderCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/render/RenderCommand.java similarity index 100% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/render/RenderCommand.java rename to gortools/src/main/java/org/gorpipe/gor/cli/render/RenderCommand.java diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/render/RenderOptions.java b/gortools/src/main/java/org/gorpipe/gor/cli/render/RenderOptions.java similarity index 100% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/render/RenderOptions.java rename to gortools/src/main/java/org/gorpipe/gor/cli/render/RenderOptions.java diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/render/ReportCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/render/ReportCommand.java similarity index 100% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/render/ReportCommand.java rename to gortools/src/main/java/org/gorpipe/gor/cli/render/ReportCommand.java diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/render/ScriptCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/render/ScriptCommand.java similarity index 100% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/render/ScriptCommand.java rename to gortools/src/main/java/org/gorpipe/gor/cli/render/ScriptCommand.java diff --git a/gortools/src/main/scala/gorsat/InputSources/Exec.scala b/gortools/src/main/scala/gorsat/InputSources/Exec.scala new file mode 100644 index 000000000..8db82e16a --- /dev/null +++ b/gortools/src/main/scala/gorsat/InputSources/Exec.scala @@ -0,0 +1,109 @@ +/* + * BEGIN_COPYRIGHT + * + * Copyright (C) 2011-2013 deCODE genetics Inc. + * Copyright (C) 2013-2019 WuXi NextCode Inc. + * All Rights Reserved. + * + * GORpipe is free software: you can redistribute it and/or modify + * it under the terms of the AFFERO GNU General Public License as published by + * the Free Software Foundation. + * + * GORpipe is distributed "AS-IS" AND WITHOUT ANY WARRANTY OF ANY KIND, + * INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, + * NON-INFRINGEMENT, OR FITNESS FOR A PARTICULAR PURPOSE. See + * the AFFERO GNU General Public License for the complete license terms. + * + * You should have received a copy of the AFFERO GNU General Public License + * along with GORpipe. If not, see + * + * END_COPYRIGHT + */ + +package gorsat.InputSources + +import gorsat.Commands.{CommandArguments, InputSourceInfo, InputSourceParsingResult} +import gorsat.Iterators.RowListIterator +import org.gorpipe.exceptions.GorParsingException +import org.gorpipe.gor.cli.GorExecCLI +import org.gorpipe.gor.model.{NoValidateRowBase, Row} +import org.gorpipe.gor.session.GorContext +import picocli.CommandLine + +import java.io.{ByteArrayOutputStream, PrintStream} +import scala.collection.mutable.ListBuffer + +/** + * Execute selected gor commands in NOR context. + */ +class Exec() extends InputSourceInfo("EXEC", CommandArguments("","", 2, 10, ignoreIllegalArguments=true), isNorCommand = true) { + + override def processArguments(context: GorContext, argString: String, iargs: Array[String], + args: Array[String]): InputSourceParsingResult = { + + val result = processExec(context, argString, iargs.slice(1, iargs.length), args) + + val header = "ChromNor\tPosNor\t" + result(0) + val myHeaderLength = header.split("\t").length + val lineList = new ListBuffer[Row]() + for (row <- result.slice(1, result.length)) { + lineList += new NoValidateRowBase("chrN\t0\t" + row, myHeaderLength) + } + + val inputSource = RowListIterator(lineList.toList) + inputSource.setHeader(header) + InputSourceParsingResult(inputSource, header, isNorContext = true) + } + + def processExec(context: GorContext, argString: String, iargs: Array[String], + args: Array[String]): Array[String] = { + + + val oldOut = System.out + val oldErr = System.err + val std_baos = new ByteArrayOutputStream() + val err_baos = new ByteArrayOutputStream() + val std_ps = new PrintStream(std_baos, true) + val err_ps = new PrintStream(std_baos, true) + var exitCode = -128 + + try { + System.setOut(std_ps) + System.setErr(err_ps) + exitCode = new CommandLine(new GorExecCLI) + .setExitCodeExceptionMapper(new CommandLine.IExitCodeExceptionMapper { + override def getExitCode(e: Throwable): Int = { + // Don't map exist codes. + throw new IllegalArgumentException(s"EXEC command: ${argString} failed: ${e.getMessage}", e) + } + }) + .setDefaultValueProvider((argSpec: CommandLine.Model.ArgSpec) => { + if (argSpec.paramLabel() == "") { + context.getSession.getProjectContext.getFileReader.getSecurityContext + } else if (argSpec.paramLabel() == "") { + context.getSession.getProjectContext.getFileReader.getCommonRoot + } else { + null + } + }) + .execute(args.slice(1, args.length): _*) + } finally { + System.setOut(oldOut) + System.setErr(oldErr) + std_ps.close() + err_ps.close() + } + if (exitCode != 0) { + throw new GorParsingException(s"EXEC command: ${argString} failed with exit code: ${exitCode} and output:\n${std_baos.toString}\n${err_baos.toString}") + } + var stdLines = std_baos.toString.stripLineEnd.split("\n") + + if (stdLines.size == 0 || !stdLines(0).startsWith("#")) { + stdLines = stdLines.prepended("Result") + } else { + stdLines(0) = stdLines(0).substring(1) + } + + stdLines + } +} diff --git a/gortools/src/main/scala/gorsat/process/GorInputSources.scala b/gortools/src/main/scala/gorsat/process/GorInputSources.scala index 9cee08ee9..b3396f58d 100644 --- a/gortools/src/main/scala/gorsat/process/GorInputSources.scala +++ b/gortools/src/main/scala/gorsat/process/GorInputSources.scala @@ -23,7 +23,7 @@ package gorsat.process import gorsat.Commands._ -import gorsat.InputSources.{Cmd, Gorif, Meta, Nor, Sql} +import gorsat.InputSources.{Cmd, Gorif, Meta, Nor, Sql, Exec} /** * Methods to register and access gor input sources. Input sources need to be registered before use. @@ -80,6 +80,7 @@ object GorInputSources { addInfo(new Sql.NorSql) addInfo(new Sql.GorSql) addInfo(new Meta) + addInfo(new Exec) } } diff --git a/gortools/src/main/scala/gorsat/process/GorPrePipe.scala b/gortools/src/main/scala/gorsat/process/GorPrePipe.scala index 560c27f8e..7619bf7ef 100644 --- a/gortools/src/main/scala/gorsat/process/GorPrePipe.scala +++ b/gortools/src/main/scala/gorsat/process/GorPrePipe.scala @@ -84,7 +84,12 @@ object GorPrePipe { || pipeSteps(0).toUpperCase.trim.startsWith("GORIF") || pipeSteps(0).toUpperCase.trim.startsWith("NORIF") || pipeSteps(0).toUpperCase.trim.startsWith("SPARK") - || pipeSteps(0).toUpperCase.trim.startsWith("SELECT"))) + || pipeSteps(0).toUpperCase.trim.startsWith("SELECT") + || pipeSteps(0).toUpperCase.trim.startsWith("META") + || pipeSteps(0).toUpperCase.trim.startsWith("EXEC") + || pipeSteps(0).toUpperCase.trim.startsWith("NORROWS") + || pipeSteps(0).toUpperCase.trim.startsWith("GORROWS") + )) "GOR " + pipeSteps(i).trim else pipeSteps(i).trim diff --git a/gortools/src/test/java/gorsat/Inputs/UTestLink.java b/gortools/src/test/java/gorsat/Inputs/UTestLink.java new file mode 100644 index 000000000..9026be77e --- /dev/null +++ b/gortools/src/test/java/gorsat/Inputs/UTestLink.java @@ -0,0 +1,158 @@ +package gorsat.Inputs; + +import gorsat.Commands.CommandParseUtilities; +import gorsat.TestUtils; +import org.gorpipe.exceptions.GorParsingException; +import org.gorpipe.gor.driver.linkfile.LinkFile; +import org.gorpipe.gor.driver.providers.stream.sources.file.FileSource; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.nio.file.Path; +import java.time.Instant; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class UTestLink { + @Rule + public TemporaryFolder temp = new TemporaryFolder(); + + @Test + public void testLinkHelp() throws Exception { + String res = TestUtils.runGorPipe("exec gor link --help "); + + assertTrue(res.split("\n")[1].split("\t")[2].startsWith("Link file management commands.")); + } + + @Test + public void testUpdateCreatesLinkFileError() throws Exception { + Path linkFile = temp.getRoot().toPath().resolve("update_test.gor.link"); + + try { + String res = TestUtils.runGorPipe("exec gor link update " + linkFile.toString() + " data/file1.gor -xxx ENTRIES_COUNT_MAX=5"); + } catch (GorParsingException e) { + String res = e.getMessage(); + assertTrue(res.contains("Unknown options: '-xxx', 'ENTRIES_COUNT_MAX=5'")); + } + LinkFile link = LinkFile.load(new FileSource(linkFile)); + assertEquals(0, link.getEntriesCount()); + } + + @Test + public void testUpdateCreatesLinkFileAndAppliesHeaders() throws Exception { + Path linkFile = temp.getRoot().toPath().resolve("update_test.gor.link"); + + String res = TestUtils.runGorPipe("exec gor link update " + linkFile.toString() + " data/file1.gor -h ENTRIES_COUNT_MAX=5"); + + LinkFile link = LinkFile.load(new FileSource(linkFile)); + assertEquals(1, link.getEntriesCount()); + assertEquals(resolve(linkFile, "data/file1.gor"), link.getLatestEntryUrl()); + assertEquals(5, link.getEntriesCountMax()); + } + + @Test + public void testUpdateCreatesLinkFileWithProjectRooot() throws Exception { + Path linkFile = temp.getRoot().toPath().resolve("update_test.gor.link"); + + String res = TestUtils.runGorPipe("exec gor link update update_test.gor.link data/file1.gor -h ENTRIES_COUNT_MAX=5", temp.getRoot().toPath().toString() , true, "dummy"); + + LinkFile link = LinkFile.load(new FileSource(linkFile)); + assertEquals(1, link.getEntriesCount()); + assertEquals(resolve(linkFile, "data/file1.gor"), link.getLatestEntryUrl()); + assertEquals(5, link.getEntriesCountMax()); + } + + @Test + public void testUpdateWithMd5AndInfo() throws Exception { + Path linkFile = temp.getRoot().toPath().resolve("update_md5_info.gor.link"); + + String res = TestUtils.runGorPipe("exec gor link update " + linkFile.toString() + " data/file1.gor -m abc123 -i 'first entry'"); + + LinkFile link = LinkFile.load(new FileSource(linkFile)); + var latest = link.getLatestEntry(); + assertEquals("abc123", latest.md5()); + assertEquals("'first entry'", latest.info()); + } + + @Test + public void testRollbackLatestEntry() throws Exception { + Path linkFile = temp.getRoot().toPath().resolve("rollback_latest.gor.link"); + + TestUtils.runGorPipe("exec gor link update " + linkFile.toString() + " data/file1.gor"); + Thread.sleep(5); + TestUtils.runGorPipe("exec gor link update " + linkFile.toString() + " data/file2.gor"); + + String res = TestUtils.runGorPipe("exec gor link rollback " + linkFile.toString()); + + LinkFile link = LinkFile.load(new FileSource(linkFile)); + assertEquals(1, link.getEntriesCount()); + assertEquals(resolve(linkFile, "data/file1.gor"), link.getLatestEntryUrl()); + } + + @Test + public void testRollbackToDate() throws Exception { + Path linkFile = temp.getRoot().toPath().resolve("rollback_date.gor.link"); + + TestUtils.runGorPipe("exec gor link update " + linkFile.toString() + " data/file1.gor"); + LinkFile first = LinkFile.load(new FileSource(linkFile)); + long firstTimestamp = first.getLatestEntry().timestamp(); + Thread.sleep(5); + TestUtils.runGorPipe("exec gor link update " + linkFile.toString() + " data/file2.gor"); + + String rollbackIso = Instant.ofEpochMilli(firstTimestamp).toString(); + String res = TestUtils.runGorPipe("exec gor link rollback " + linkFile.toString() + " -d " + rollbackIso); + + LinkFile link = LinkFile.load(new FileSource(linkFile)); + assertEquals(1, link.getEntriesCount()); + assertEquals(resolve(linkFile, "data/file1.gor"), link.getLatestEntryUrl()); + } + + @Test + public void testResolveLatestEntry() throws Exception { + Path linkFile = temp.getRoot().toPath().resolve("resolve_latest.gor.link"); + + TestUtils.runGorPipe("exec gor link update " + linkFile.toString() + " data/file1.gor"); + Thread.sleep(5); + TestUtils.runGorPipe("exec gor link update " + linkFile.toString() + " data/file2.gor"); + + String res = TestUtils.runGorPipe("exec gor link resolve " + linkFile); + + assertEquals(resolve(linkFile, "data/file2.gor"), res.split("\n")[1].split("\t")[2]); + } + + @Test + public void testResolveSpecificDate() throws Exception { + Path linkFile = temp.getRoot().toPath().resolve("resolve_date.gor.link"); + + TestUtils.runGorPipe("exec gor link update " + linkFile.toString() + " data/file1.gor"); + LinkFile first = LinkFile.load(new FileSource(linkFile)); + long firstTimestamp = first.getLatestEntry().timestamp(); + Thread.sleep(5); + TestUtils.runGorPipe("exec gor link update " + linkFile.toString() + " data/file2.gor"); + + String res = TestUtils.runGorPipe("exec gor link resolve " + linkFile.toString() + " -d " + Instant.ofEpochMilli(firstTimestamp).toString()); + + assertEquals(resolve(linkFile, "data/file1.gor"), res.split("\n")[1].split("\t")[2]); + } + + @Test + public void testResolveFullEntry() throws Exception { + Path linkFile = temp.getRoot().toPath().resolve("resolve_full.gor.link"); + + TestUtils.runGorPipe("exec gor link update " + linkFile.toString() + " data/file1.gor"); + Thread.sleep(5); + TestUtils.runGorPipe("exec gor link update " + linkFile.toString() + " data/file2.gor"); + + String expectedEntry = LinkFile.load(new FileSource(linkFile)).getLatestEntry().format(); + + String res = TestUtils.runGorPipe("exec gor link resolve " + linkFile.toString() + " -f"); + + assertEquals(expectedEntry.replace('\t', ' '), CommandParseUtilities.quoteSafeSplit(res.split("\n")[1], '\t')[2]); + } + + private String resolve(Path linkFile, String relative) { + return linkFile.getParent().resolve(relative).toAbsolutePath().normalize().toString(); + } +} diff --git a/model/src/main/scala/gorsat/gorsatDynIterator.scala b/model/src/main/scala/gorsat/gorsatDynIterator.scala index c3d49fd8f..3d6716cdf 100644 --- a/model/src/main/scala/gorsat/gorsatDynIterator.scala +++ b/model/src/main/scala/gorsat/gorsatDynIterator.scala @@ -58,6 +58,8 @@ def addStartSelector(cmd : String, seekChr : String, seekPos : Int, endPos : Int else if (cmd.toLowerCase.startsWith("norrows ")) return cmd else if (cmd.toLowerCase.startsWith("sdl ")) return cmd else if (cmd.toLowerCase.startsWith("norsql ")) return cmd + else if (cmd.toLowerCase.startsWith("meta ")) return cmd + else if (cmd.toLowerCase.startsWith("exec ")) return cmd var cmd2 = cmd val lcmd = cmd2.toLowerCase