From 68df1a12f662a232a5f8944c9022d0600415fb38 Mon Sep 17 00:00:00 2001 From: Gisli Magnusson Date: Tue, 18 Nov 2025 16:02:44 +0000 Subject: [PATCH 01/17] feat(ENGKNOW-2892): Add command line support for working with link files. --- .../main/java/org/gorpipe/gor/cli/GorCLI.java | 4 +- .../org/gorpipe/gor/cli/link/LinkCommand.java | 17 ++ .../gor/cli/link/LinkRollbackCommand.java | 50 ++++++ .../cli/link/LinkStreamSourceProvider.java | 26 +++ .../gor/cli/link/LinkUpdateCommand.java | 62 +++++++ .../gorpipe/gor/cli/link/LinkCommandTest.java | 154 ++++++++++++++++++ .../gorpipe/gor/driver/linkfile/LinkFile.java | 47 +++++- .../gor/driver/linkfile/LinkFileEntryV1.java | 6 +- 8 files changed, 356 insertions(+), 10 deletions(-) create mode 100644 gorscripts/src/main/java/org/gorpipe/gor/cli/link/LinkCommand.java create mode 100644 gorscripts/src/main/java/org/gorpipe/gor/cli/link/LinkRollbackCommand.java create mode 100644 gorscripts/src/main/java/org/gorpipe/gor/cli/link/LinkStreamSourceProvider.java create mode 100644 gorscripts/src/main/java/org/gorpipe/gor/cli/link/LinkUpdateCommand.java create mode 100644 gorscripts/src/test/java/org/gorpipe/gor/cli/link/LinkCommandTest.java diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/GorCLI.java b/gorscripts/src/main/java/org/gorpipe/gor/cli/GorCLI.java index 84d7b40a..a3e201f5 100644 --- a/gorscripts/src/main/java/org/gorpipe/gor/cli/GorCLI.java +++ b/gorscripts/src/main/java/org/gorpipe/gor/cli/GorCLI.java @@ -26,6 +26,7 @@ import org.gorpipe.gor.cli.help.HelpCommand; import org.gorpipe.gor.cli.index.IndexCommand; import org.gorpipe.gor.cli.info.InfoCommand; +import org.gorpipe.gor.cli.link.LinkCommand; import org.gorpipe.gor.cli.manager.ManagerCommand; import org.gorpipe.gor.cli.migrator.FolderMigratorCommand; import org.gorpipe.gor.cli.query.QueryCommand; @@ -38,7 +39,8 @@ version="version 1.0", description = "Command line interface for gor query language and processes.", subcommands = {QueryCommand.class, HelpCommand.class, ManagerCommand.class, IndexCommand.class, - CacheCommand.class, RenderCommand.class, InfoCommand.class, FolderMigratorCommand.class}) + CacheCommand.class, RenderCommand.class, InfoCommand.class, FolderMigratorCommand.class, + LinkCommand.class}) public class GorCLI extends HelpOptions implements Runnable { public static void main(String[] args) { GorLogbackUtil.initLog("gor"); diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/link/LinkCommand.java b/gorscripts/src/main/java/org/gorpipe/gor/cli/link/LinkCommand.java new file mode 100644 index 00000000..c83be67c --- /dev/null +++ b/gorscripts/src/main/java/org/gorpipe/gor/cli/link/LinkCommand.java @@ -0,0 +1,17 @@ +package org.gorpipe.gor.cli.link; + +import org.gorpipe.gor.cli.HelpOptions; +import picocli.CommandLine; + +@SuppressWarnings("squid:S106") +@CommandLine.Command(name = "link", + description = "Manage link files (create, update, rollback).", + header = "Link file management commands.", + subcommands = {LinkUpdateCommand.class, LinkRollbackCommand.class, LinkResolveCommand.class}) +public class LinkCommand extends HelpOptions implements Runnable { + + @Override + public void run() { + CommandLine.usage(this, System.err); + } +} diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/link/LinkRollbackCommand.java b/gorscripts/src/main/java/org/gorpipe/gor/cli/link/LinkRollbackCommand.java new file mode 100644 index 00000000..4a239fb7 --- /dev/null +++ b/gorscripts/src/main/java/org/gorpipe/gor/cli/link/LinkRollbackCommand.java @@ -0,0 +1,50 @@ +package org.gorpipe.gor.cli.link; + +import java.io.IOException; +import java.time.Instant; + +import org.gorpipe.gor.driver.linkfile.LinkFile; +import org.gorpipe.util.DateUtils; + +import picocli.CommandLine; + +@CommandLine.Command(name = "rollback", + description = "Rollback the latest entry or rollback entries newer than a given date.") +public class LinkRollbackCommand implements Runnable { + + @CommandLine.Parameters(index = "0", paramLabel = "LINK_FILE", description = "Path to the link file to rollback.") + private String linkFilePath; + + @CommandLine.Option(names = {"-d", "--date"}, paramLabel = "DATE", + description = "ISO-8601 date/time or epoch milliseconds to rollback to (entries newer than this are removed).") + private String rollbackDate; + + @Override + public void run() { + var normalizedLinkPath = LinkFile.validateAndUpdateLinkFileName(linkFilePath); + try { + var linkFile = LinkFile.load(LinkStreamSourceProvider.resolve(normalizedLinkPath, true, this)); + boolean changed = rollbackDate == null ? linkFile.rollbackLatestEntry() : linkFile.rollbackToTimestamp(parseDate(rollbackDate)); + if (!changed) { + throw new CommandLine.ParameterException(new CommandLine(this), + "No entries were removed. Link file may already be at the requested state."); + } + linkFile.save(); + System.err.printf("Rolled back link file %s%n", normalizedLinkPath); + } catch (IOException e) { + throw new CommandLine.ExecutionException(new CommandLine(this), + "Failed to load link file: " + normalizedLinkPath, e); + } + } + + private long parseDate(String dateValue) { + try { + Instant instant = DateUtils.parseDateISOEpoch(dateValue, true); + return instant.toEpochMilli(); + } catch (Exception e) { + throw new CommandLine.ParameterException(new CommandLine(this), + "Invalid date value: " + dateValue, e); + } + } +} + diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/link/LinkStreamSourceProvider.java b/gorscripts/src/main/java/org/gorpipe/gor/cli/link/LinkStreamSourceProvider.java new file mode 100644 index 00000000..bfde54a8 --- /dev/null +++ b/gorscripts/src/main/java/org/gorpipe/gor/cli/link/LinkStreamSourceProvider.java @@ -0,0 +1,26 @@ +package org.gorpipe.gor.cli.link; + +import org.gorpipe.gor.driver.providers.stream.sources.StreamSource; +import org.gorpipe.gor.model.DriverBackedFileReader; +import picocli.CommandLine; + +final class LinkStreamSourceProvider { + + private LinkStreamSourceProvider() { + } + + static StreamSource resolve(String linkPath, boolean writeable, Object commandInstance) { + var fileReader = new DriverBackedFileReader("", null); + var dataSource = fileReader.resolveUrl(linkPath, writeable); + if (dataSource == null) { + throw new CommandLine.ExecutionException(new CommandLine(commandInstance), + "Could not resolve link file path: " + linkPath); + } + if (dataSource instanceof StreamSource streamSource) { + return streamSource; + } + throw new CommandLine.ExecutionException(new CommandLine(commandInstance), + "Link path is not stream compatible: " + linkPath); + } +} + diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/link/LinkUpdateCommand.java b/gorscripts/src/main/java/org/gorpipe/gor/cli/link/LinkUpdateCommand.java new file mode 100644 index 00000000..c6247e3d --- /dev/null +++ b/gorscripts/src/main/java/org/gorpipe/gor/cli/link/LinkUpdateCommand.java @@ -0,0 +1,62 @@ +package org.gorpipe.gor.cli.link; + +import java.io.IOException; +import java.util.LinkedHashMap; +import java.util.Map; + +import org.gorpipe.gor.driver.linkfile.LinkFile; +import org.gorpipe.util.Strings; + +import picocli.CommandLine; + +@CommandLine.Command(name = "update", + description = "Append a new entry to a link file, creating the file if needed.") +public class LinkUpdateCommand implements Runnable { + + @CommandLine.Parameters(index = "0", paramLabel = "LINK_FILE", description = "Path to the link file to update.") + private String linkFilePath; + + @CommandLine.Parameters(index = "1", paramLabel = "LINK_VALUE", description = "Value to add to the link file (file path, URL or query).") + private String linkValue; + + @CommandLine.Option(names = {"-m", "--md5"}, paramLabel = "MD5", + description = "MD5 checksum to associate with the new link entry.") + private String entryMd5; + + @CommandLine.Option(names = {"-i", "--info"}, paramLabel = "INFO", + description = "Free-form info string to store with the new link entry.") + private String entryInfo; + + @SuppressWarnings("MismatchedQueryAndUpdateOfCollection") + @CommandLine.Option(names = {"-h", "--header"}, paramLabel = "KEY=VALUE", + description = "Header property to upsert in the link file metadata. Repeatable.", + mapFallbackValue = "") + private final Map headerParams = new LinkedHashMap<>(); + + @Override + public void run() { + var normalizedLinkPath = LinkFile.validateAndUpdateLinkFileName(linkFilePath); + try { + var linkFile = LinkFile.load(LinkStreamSourceProvider.resolve(normalizedLinkPath, true, this)); + applyHeaders(linkFile); + linkFile.appendEntry(linkValue, entryMd5, entryInfo); + linkFile.save(); + System.out.printf("Updated link file %s with %s%n", normalizedLinkPath, linkValue); + } catch (IOException e) { + throw new CommandLine.ExecutionException(new CommandLine(this), + "Failed to load or create link file: " + normalizedLinkPath, e); + } + } + + private void applyHeaders(LinkFile linkFile) { + for (var entry : headerParams.entrySet()) { + var key = entry.getKey(); + var value = entry.getValue(); + if (Strings.isNullOrEmpty(key)) { + continue; + } + linkFile.getMeta().setProperty(key.trim().toUpperCase(), value != null ? value.trim() : ""); + } + } +} + diff --git a/gorscripts/src/test/java/org/gorpipe/gor/cli/link/LinkCommandTest.java b/gorscripts/src/test/java/org/gorpipe/gor/cli/link/LinkCommandTest.java new file mode 100644 index 00000000..d8a519fb --- /dev/null +++ b/gorscripts/src/test/java/org/gorpipe/gor/cli/link/LinkCommandTest.java @@ -0,0 +1,154 @@ +package org.gorpipe.gor.cli.link; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.file.Path; +import java.time.Instant; + +import org.gorpipe.gor.driver.linkfile.LinkFile; +import org.gorpipe.gor.driver.providers.stream.sources.file.FileSource; +import static org.junit.Assert.assertEquals; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import picocli.CommandLine; + +public class LinkCommandTest { + + @Rule + public TemporaryFolder temp = new TemporaryFolder(); + + @Test + public void testUpdateCreatesLinkFileAndAppliesHeaders() throws Exception { + Path linkFile = temp.getRoot().toPath().resolve("update_test.gor.link"); + CommandLine cmd = new CommandLine(new LinkCommand()); + + int exitCode = cmd.execute("update", linkFile.toString(), "data/file1.gor", "-h", "ENTRIES_COUNT_MAX=5"); + assertEquals(0, exitCode); + + LinkFile link = LinkFile.load(new FileSource(linkFile)); + assertEquals(1, link.getEntriesCount()); + assertEquals(resolve(linkFile, "data/file1.gor"), link.getLatestEntryUrl()); + assertEquals(5, link.getEntriesCountMax()); + } + + @Test + public void testUpdateWithMd5AndInfo() throws Exception { + Path linkFile = temp.getRoot().toPath().resolve("update_md5_info.gor.link"); + CommandLine cmd = new CommandLine(new LinkCommand()); + + int exitCode = cmd.execute("update", linkFile.toString(), "data/file1.gor", + "-m", "abc123", "-i", "first entry"); + assertEquals(0, exitCode); + + LinkFile link = LinkFile.load(new FileSource(linkFile)); + var latest = link.getLatestEntry(); + assertEquals("abc123", latest.md5()); + assertEquals("first entry", latest.info()); + } + + @Test + public void testRollbackLatestEntry() throws Exception { + Path linkFile = temp.getRoot().toPath().resolve("rollback_latest.gor.link"); + CommandLine cmd = new CommandLine(new LinkCommand()); + + cmd.execute("update", linkFile.toString(), "data/file1.gor"); + Thread.sleep(5); + cmd.execute("update", linkFile.toString(), "data/file2.gor"); + + int exitCode = cmd.execute("rollback", linkFile.toString()); + assertEquals(0, exitCode); + + LinkFile link = LinkFile.load(new FileSource(linkFile)); + assertEquals(1, link.getEntriesCount()); + assertEquals(resolve(linkFile, "data/file1.gor"), link.getLatestEntryUrl()); + } + + @Test + public void testRollbackToDate() throws Exception { + Path linkFile = temp.getRoot().toPath().resolve("rollback_date.gor.link"); + CommandLine cmd = new CommandLine(new LinkCommand()); + + cmd.execute("update", linkFile.toString(), "data/file1.gor"); + LinkFile first = LinkFile.load(new FileSource(linkFile)); + long firstTimestamp = first.getLatestEntry().timestamp(); + + Thread.sleep(5); + cmd.execute("update", linkFile.toString(), "data/file2.gor"); + + String rollbackIso = Instant.ofEpochMilli(firstTimestamp).toString(); + int exitCode = cmd.execute("rollback", linkFile.toString(), "-d", rollbackIso); + assertEquals(0, exitCode); + + LinkFile link = LinkFile.load(new FileSource(linkFile)); + assertEquals(1, link.getEntriesCount()); + assertEquals(resolve(linkFile, "data/file1.gor"), link.getLatestEntryUrl()); + } + + @Test + public void testResolveLatestEntry() throws Exception { + Path linkFile = temp.getRoot().toPath().resolve("resolve_latest.gor.link"); + CommandLine cmd = new CommandLine(new LinkCommand()); + + cmd.execute("update", linkFile.toString(), "data/file1.gor"); + Thread.sleep(5); + cmd.execute("update", linkFile.toString(), "data/file2.gor"); + + String resolved = executeAndCapture(cmd, "resolve", linkFile.toString()); + assertEquals(resolve(linkFile, "data/file2.gor"), resolved); + } + + @Test + public void testResolveSpecificDate() throws Exception { + Path linkFile = temp.getRoot().toPath().resolve("resolve_date.gor.link"); + CommandLine cmd = new CommandLine(new LinkCommand()); + + cmd.execute("update", linkFile.toString(), "data/file1.gor"); + long firstTimestamp = LinkFile.load(new FileSource(linkFile)).getLatestEntry().timestamp(); + Thread.sleep(5); + cmd.execute("update", linkFile.toString(), "data/file2.gor"); + + String resolved = executeAndCapture(cmd, "resolve", linkFile.toString(), + "-d", Instant.ofEpochMilli(firstTimestamp).toString()); + assertEquals(resolve(linkFile, "data/file1.gor"), resolved); + } + + @Test + public void testResolveFullEntry() throws Exception { + Path linkFile = temp.getRoot().toPath().resolve("resolve_full.gor.link"); + CommandLine cmd = new CommandLine(new LinkCommand()); + + cmd.execute("update", linkFile.toString(), "data/file1.gor"); + Thread.sleep(5); + cmd.execute("update", linkFile.toString(), "data/file2.gor"); + + String expectedEntry = LinkFile.load(new FileSource(linkFile)).getLatestEntry().format(); + String resolved = executeAndCapture(cmd, "resolve", linkFile.toString(), "-f"); + assertEquals(expectedEntry, resolved); + } + + private String resolve(Path linkFile, String relative) { + return linkFile.getParent().resolve(relative).toAbsolutePath().normalize().toString(); + } + + private String executeAndCapture(CommandLine cmd, String... args) { + PrintStream originalOut = System.out; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + System.setOut(new PrintStream(baos, true)); + try { + int exitCode = cmd.execute(args); + assertEquals(0, exitCode); + } finally { + System.setOut(originalOut); + } + String output = baos.toString(); + if (output.endsWith("\r\n")) { + output = output.substring(0, output.length() - 2); + } else if (output.endsWith("\n") || output.endsWith("\r")) { + output = output.substring(0, output.length() - 1); + } + return output; + } +} + diff --git a/model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFile.java b/model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFile.java index b7f94d3c..1e5c34e6 100644 --- a/model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFile.java +++ b/model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFile.java @@ -1,8 +1,11 @@ package org.gorpipe.gor.driver.linkfile; -import com.github.benmanes.caffeine.cache.Cache; -import com.github.benmanes.caffeine.cache.Caffeine; -import com.google.common.util.concurrent.UncheckedExecutionException; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.List; +import java.util.concurrent.TimeUnit; + import org.gorpipe.exceptions.GorResourceException; import org.gorpipe.gor.driver.meta.SourceReference; import org.gorpipe.gor.driver.providers.stream.StreamUtils; @@ -12,11 +15,9 @@ import org.gorpipe.gor.util.DataUtil; import org.gorpipe.util.Strings; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.util.List; -import java.util.concurrent.TimeUnit; +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; +import com.google.common.util.concurrent.UncheckedExecutionException; /** * Class to work with link files, read, write and access metadata. @@ -207,6 +208,34 @@ public LinkFile appendMeta(String meta) { return this; } + /** + * Remove the latest entry, if any. + * + * @return true if an entry was removed, otherwise false. + */ + public boolean rollbackLatestEntry() { + if (entries.isEmpty()) { + return false; + } + entries.remove(entries.size() - 1); + return true; + } + + /** + * Remove entries that are newer than the provided timestamp. + * + * @param timestamp the timestamp to rollback to (inclusive) + * @return true if one or more entries were removed, otherwise false. + */ + public boolean rollbackToTimestamp(long timestamp) { + boolean removed = false; + while (!entries.isEmpty() && entries.get(entries.size() - 1).timestamp() > timestamp) { + entries.remove(entries.size() - 1); + removed = true; + } + return removed; + } + public void save() { save(-1); } @@ -220,6 +249,8 @@ public void save(long timestamp) { } private void save(OutputStream os, long timestamp) { + meta.setProperty(meta.HEADER_SERIAL_KEY, Integer.toString(Integer.parseInt(meta.getProperty(meta.HEADER_SERIAL_KEY, "0")) + 1)); + var content = new StringBuilder(getHeader()); if (!entries.isEmpty()) { diff --git a/model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFileEntryV1.java b/model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFileEntryV1.java index a8be5d8b..14dbff69 100644 --- a/model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFileEntryV1.java +++ b/model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFileEntryV1.java @@ -70,6 +70,10 @@ private static LinkFileEntryV1 parseLine(String line) { } public String format() { - return url + "\t" + Instant.ofEpochMilli(timestamp) + "\t" + md5 + "\t" + serial+ "\t" + info; + return url + + "\t" + Instant.ofEpochMilli(timestamp) + + "\t" + (md5 != null ? md5 : "") + + "\t" + serial + + "\t" + (info != null ? info : ""); } } From e4452877289cbc6cdcfda0f6bf3609f6f13b519c Mon Sep 17 00:00:00 2001 From: Gisli Magnusson Date: Tue, 18 Nov 2025 16:03:49 +0000 Subject: [PATCH 02/17] feat(ENGKNOW-2892): Add command line support for working with link files. --- .../gor/cli/link/LinkResolveCommand.java | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 gorscripts/src/main/java/org/gorpipe/gor/cli/link/LinkResolveCommand.java diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/link/LinkResolveCommand.java b/gorscripts/src/main/java/org/gorpipe/gor/cli/link/LinkResolveCommand.java new file mode 100644 index 00000000..23a22671 --- /dev/null +++ b/gorscripts/src/main/java/org/gorpipe/gor/cli/link/LinkResolveCommand.java @@ -0,0 +1,67 @@ +package org.gorpipe.gor.cli.link; + +import java.io.IOException; +import java.time.Instant; + +import org.gorpipe.gor.driver.linkfile.LinkFile; +import org.gorpipe.util.DateUtils; +import org.gorpipe.util.Strings; + +import picocli.CommandLine; + +@CommandLine.Command(name = "resolve", + description = "Resolve a link file to the entry active at the current or given time.") +public class LinkResolveCommand implements Runnable { + + @CommandLine.Parameters(index = "0", paramLabel = "LINK_FILE", + description = "Path to the link file to resolve.") + private String linkFilePath; + + @CommandLine.Option(names = {"-d", "--date"}, paramLabel = "DATE", + description = "ISO-8601 date/time or epoch milliseconds to resolve at (default: now).") + private String resolveDate; + + @CommandLine.Option(names = {"-f", "--full-entry"}, + description = "Return the full link file entry instead of only the resolved URL.") + private boolean returnFullEntry; + + @Override + public void run() { + var normalizedLinkPath = LinkFile.validateAndUpdateLinkFileName(linkFilePath); + try { + var linkFile = LinkFile.load(LinkStreamSourceProvider.resolve(normalizedLinkPath, true, this)); + long timestamp = resolveDate == null ? System.currentTimeMillis() : parseDate(resolveDate); + var entry = linkFile.getEntry(timestamp); + if (entry == null) { + throw new CommandLine.ParameterException(new CommandLine(this), + "No link entry found for the requested time."); + } + String output; + if (returnFullEntry) { + output = entry.format(); + } else { + var resolved = linkFile.getEntryUrl(timestamp); + if (Strings.isNullOrEmpty(resolved)) { + throw new CommandLine.ParameterException(new CommandLine(this), + "No link entry found for the requested time."); + } + output = resolved; + } + System.out.println(output); + } catch (IOException e) { + throw new CommandLine.ExecutionException(new CommandLine(this), + "Failed to load link file: " + normalizedLinkPath, e); + } + } + + private long parseDate(String dateValue) { + try { + Instant instant = DateUtils.parseDateISOEpoch(dateValue, true); + return instant.toEpochMilli(); + } catch (Exception e) { + throw new CommandLine.ParameterException(new CommandLine(this), + "Invalid date value: " + dateValue, e); + } + } +} + From 0c0abda161c9d2970a9f5e42109d95ca2d1bc250 Mon Sep 17 00:00:00 2001 From: Gisli Magnusson Date: Tue, 18 Nov 2025 16:46:49 +0000 Subject: [PATCH 03/17] feat(ENGKNOW-2892): Add command line support for working with link files. --- gortools/src/test/java/gorsat/UTestGorWrite.java | 6 +++--- .../java/org/gorpipe/gor/driver/linkfile/LinkFileTest.java | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/gortools/src/test/java/gorsat/UTestGorWrite.java b/gortools/src/test/java/gorsat/UTestGorWrite.java index b60a67ba..ce1e669b 100644 --- a/gortools/src/test/java/gorsat/UTestGorWrite.java +++ b/gortools/src/test/java/gorsat/UTestGorWrite.java @@ -71,6 +71,7 @@ public void setupTest() throws IOException { var meta = new LinkFileMeta(); meta.setProperty(BaseMeta.HEADER_VERSION_KEY, "1"); + meta.setProperty(BaseMeta.HEADER_SERIAL_KEY, "1"); defaultV1LinkFileHeader = meta.formatHeader(); } @@ -136,9 +137,8 @@ public void testWritePathWithExistingVersionedLinkFile() throws IOException { Files.writeString(workDirPath.resolve("dbsnp3.gor.link"), defaultV1LinkFileHeader + workDirPath.resolve("dbsnp.gor").toString() + "\n"); TestUtils.runGorPipe("gor dbsnp.gor | write dbsnp2.gor -link dbsnp3.gor", "-gorroot", workDirPath.toString()); - Assert.assertTrue(Files.readString(workDirPath.resolve("dbsnp3.gor.link")).startsWith( - defaultV1LinkFileHeader - + workDirPath.resolve("dbsnp.gor") + "\t1970-01-01T00:00:00Z\t\t0\t\n" + Assert.assertTrue(Files.readString(workDirPath.resolve("dbsnp3.gor.link")).contains( + workDirPath.resolve("dbsnp.gor") + "\t1970-01-01T00:00:00Z\t\t0\t\n" + workDirPath.resolve("dbsnp2.gor") + "\t")); } diff --git a/model/src/test/java/org/gorpipe/gor/driver/linkfile/LinkFileTest.java b/model/src/test/java/org/gorpipe/gor/driver/linkfile/LinkFileTest.java index 6a3836d4..b64d7a9a 100644 --- a/model/src/test/java/org/gorpipe/gor/driver/linkfile/LinkFileTest.java +++ b/model/src/test/java/org/gorpipe/gor/driver/linkfile/LinkFileTest.java @@ -22,7 +22,7 @@ public class LinkFileTest { private StreamSource mockSource; private final String v1LinkFileContent = """ - ## SERIAL = 0 + ## SERIAL = 1 ## VERSION = 1 #FILE\tTIMESTAMP\tMD5\tSERIAL\tINFO source/v1/ver1.gorz\t2024-12-15T11:21:30.790Z\tABCDEAF13422\t1\t @@ -117,7 +117,7 @@ public void testSaveLinkFileV1ToV1() throws IOException { linkFile.appendEntry(simpleFile, "NEWMD5SUM"); linkFile.save(); String savedContent = Files.readString(linkPath); - assertTrue(savedContent.startsWith(v1LinkFileContent)); + assertTrue(savedContent.startsWith("## SERIAL = 2")); assertTrue(savedContent.contains(simpleFile)); } From ea197947a68a90f466ffd34e17d0b3c114794374 Mon Sep 17 00:00:00 2001 From: Gisli Magnusson Date: Mon, 1 Dec 2025 19:08:15 -0500 Subject: [PATCH 04/17] feat(ENGKNOW-2892): Add gor query support for link files in gor server. --- .../main/java/org/gorpipe/gor/cli/GorCLI.java | 4 +- .../gorpipe/gor/cli/query/QueryCommand.java | 7 +- .../gorpipe/gor/cli/UTestGorCliManager.java | 36 ++-- gortools/build.gradle | 2 + .../gorpipe/gor/cli/FormattingOptions.java | 0 .../java/org/gorpipe/gor/cli/GorExecCLI.java | 77 ++++++++ .../java/org/gorpipe/gor/cli/HelpOptions.java | 1 + .../gorpipe/gor/cli/cache/AnalysisResult.java | 0 .../gorpipe/gor/cli/cache/CacheCommand.java | 0 .../gorpipe/gor/cli/cache/FilterOptions.java | 0 .../gorpipe/gor/cli/cache/PurgeCommand.java | 0 .../gorpipe/gor/cli/cache/TouchCommand.java | 0 .../org/gorpipe/gor/cli/help/HelpCommand.java | 2 +- .../gorpipe/gor/cli/index/IndexCommand.java | 0 .../gorpipe/gor/cli/info/CommandsCommand.java | 0 .../org/gorpipe/gor/cli/info/InfoCommand.java | 0 .../gorpipe/gor/cli/info/InputsCommand.java | 0 .../gorpipe/gor/cli/info/MacrosCommand.java | 0 .../org/gorpipe/gor/cli/link/LinkCommand.java | 12 ++ .../gor/cli/link/LinkResolveCommand.java | 13 +- .../gor/cli/link/LinkRollbackCommand.java | 13 +- .../cli/link/LinkStreamSourceProvider.java | 4 +- .../gor/cli/link/LinkUpdateCommand.java | 17 +- .../gor/cli/manager/BucketizeCommand.java | 2 - .../cli/manager/CommandBucketizeOptions.java | 0 .../gor/cli/manager/DeleteBucketCommand.java | 2 - .../gor/cli/manager/DeleteCommand.java | 0 .../gor/cli/manager/FilterOptions.java | 0 .../gor/cli/manager/InsertCommand.java | 0 .../gor/cli/manager/ManagerCommand.java | 0 .../gor/cli/manager/ManagerOptions.java | 0 .../gor/cli/manager/MultiInsertCommand.java | 0 .../gor/cli/manager/SelectCommand.java | 0 .../gorpipe/gor/cli/manager/TestCommand.java | 0 .../gor/cli/manager/TestIsLockCommand.java | 0 .../gor/cli/manager/TestReadLockCommand.java | 0 .../gor/cli/manager/TestWriteLockCommand.java | 0 .../gorpipe/gor/cli/render/RenderCommand.java | 0 .../gorpipe/gor/cli/render/RenderOptions.java | 0 .../gorpipe/gor/cli/render/ReportCommand.java | 0 .../gorpipe/gor/cli/render/ScriptCommand.java | 0 .../main/scala/gorsat/InputSources/Exec.scala | 100 +++++++++++ .../gorsat/process/GorInputSources.scala | 3 +- .../test/java/gorsat/Inputs/UTestLink.java | 164 ++++++++++++++++++ 44 files changed, 410 insertions(+), 49 deletions(-) rename {gorscripts => gortools}/src/main/java/org/gorpipe/gor/cli/FormattingOptions.java (100%) create mode 100644 gortools/src/main/java/org/gorpipe/gor/cli/GorExecCLI.java rename {gorscripts => gortools}/src/main/java/org/gorpipe/gor/cli/HelpOptions.java (95%) rename {gorscripts => gortools}/src/main/java/org/gorpipe/gor/cli/cache/AnalysisResult.java (100%) rename {gorscripts => gortools}/src/main/java/org/gorpipe/gor/cli/cache/CacheCommand.java (100%) rename {gorscripts => gortools}/src/main/java/org/gorpipe/gor/cli/cache/FilterOptions.java (100%) rename {gorscripts => gortools}/src/main/java/org/gorpipe/gor/cli/cache/PurgeCommand.java (100%) rename {gorscripts => gortools}/src/main/java/org/gorpipe/gor/cli/cache/TouchCommand.java (100%) rename {gorscripts => gortools}/src/main/java/org/gorpipe/gor/cli/help/HelpCommand.java (97%) rename {gorscripts => gortools}/src/main/java/org/gorpipe/gor/cli/index/IndexCommand.java (100%) rename {gorscripts => gortools}/src/main/java/org/gorpipe/gor/cli/info/CommandsCommand.java (100%) rename {gorscripts => gortools}/src/main/java/org/gorpipe/gor/cli/info/InfoCommand.java (100%) rename {gorscripts => gortools}/src/main/java/org/gorpipe/gor/cli/info/InputsCommand.java (100%) rename {gorscripts => gortools}/src/main/java/org/gorpipe/gor/cli/info/MacrosCommand.java (100%) rename {gorscripts => gortools}/src/main/java/org/gorpipe/gor/cli/link/LinkCommand.java (65%) rename {gorscripts => gortools}/src/main/java/org/gorpipe/gor/cli/link/LinkResolveCommand.java (92%) rename {gorscripts => gortools}/src/main/java/org/gorpipe/gor/cli/link/LinkRollbackCommand.java (90%) rename {gorscripts => gortools}/src/main/java/org/gorpipe/gor/cli/link/LinkStreamSourceProvider.java (78%) rename {gorscripts => gortools}/src/main/java/org/gorpipe/gor/cli/link/LinkUpdateCommand.java (90%) rename {gorscripts => gortools}/src/main/java/org/gorpipe/gor/cli/manager/BucketizeCommand.java (98%) rename {gorscripts => gortools}/src/main/java/org/gorpipe/gor/cli/manager/CommandBucketizeOptions.java (100%) rename {gorscripts => gortools}/src/main/java/org/gorpipe/gor/cli/manager/DeleteBucketCommand.java (96%) rename {gorscripts => gortools}/src/main/java/org/gorpipe/gor/cli/manager/DeleteCommand.java (100%) rename {gorscripts => gortools}/src/main/java/org/gorpipe/gor/cli/manager/FilterOptions.java (100%) rename {gorscripts => gortools}/src/main/java/org/gorpipe/gor/cli/manager/InsertCommand.java (100%) rename {gorscripts => gortools}/src/main/java/org/gorpipe/gor/cli/manager/ManagerCommand.java (100%) rename {gorscripts => gortools}/src/main/java/org/gorpipe/gor/cli/manager/ManagerOptions.java (100%) rename {gorscripts => gortools}/src/main/java/org/gorpipe/gor/cli/manager/MultiInsertCommand.java (100%) rename {gorscripts => gortools}/src/main/java/org/gorpipe/gor/cli/manager/SelectCommand.java (100%) rename {gorscripts => gortools}/src/main/java/org/gorpipe/gor/cli/manager/TestCommand.java (100%) rename {gorscripts => gortools}/src/main/java/org/gorpipe/gor/cli/manager/TestIsLockCommand.java (100%) rename {gorscripts => gortools}/src/main/java/org/gorpipe/gor/cli/manager/TestReadLockCommand.java (100%) rename {gorscripts => gortools}/src/main/java/org/gorpipe/gor/cli/manager/TestWriteLockCommand.java (100%) rename {gorscripts => gortools}/src/main/java/org/gorpipe/gor/cli/render/RenderCommand.java (100%) rename {gorscripts => gortools}/src/main/java/org/gorpipe/gor/cli/render/RenderOptions.java (100%) rename {gorscripts => gortools}/src/main/java/org/gorpipe/gor/cli/render/ReportCommand.java (100%) rename {gorscripts => gortools}/src/main/java/org/gorpipe/gor/cli/render/ScriptCommand.java (100%) create mode 100644 gortools/src/main/scala/gorsat/InputSources/Exec.scala create mode 100644 gortools/src/test/java/gorsat/Inputs/UTestLink.java diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/GorCLI.java b/gorscripts/src/main/java/org/gorpipe/gor/cli/GorCLI.java index a3e201f5..314f2ec1 100644 --- a/gorscripts/src/main/java/org/gorpipe/gor/cli/GorCLI.java +++ b/gorscripts/src/main/java/org/gorpipe/gor/cli/GorCLI.java @@ -41,10 +41,10 @@ subcommands = {QueryCommand.class, HelpCommand.class, ManagerCommand.class, IndexCommand.class, CacheCommand.class, RenderCommand.class, InfoCommand.class, FolderMigratorCommand.class, LinkCommand.class}) -public class GorCLI extends HelpOptions implements Runnable { +public class GorCLI extends GorExecCLI implements Runnable { public static void main(String[] args) { GorLogbackUtil.initLog("gor"); - CommandLine cmd = new CommandLine(new GorCLI()); + CommandLine cmd = new CommandLine(new GorExecCLI()); cmd.parseWithHandlers( new CommandLine.RunLast(), CommandLine.defaultExceptionHandler().andExit(-1), diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/query/QueryCommand.java b/gorscripts/src/main/java/org/gorpipe/gor/cli/query/QueryCommand.java index 164ce9cc..a53d6553 100644 --- a/gorscripts/src/main/java/org/gorpipe/gor/cli/query/QueryCommand.java +++ b/gorscripts/src/main/java/org/gorpipe/gor/cli/query/QueryCommand.java @@ -63,9 +63,6 @@ public class QueryCommand extends HelpOptions implements Runnable{ @CommandLine.Option(names={"-d","--cachedir"}, description = "Path to cache directory for the current gor query.") private Path cacheDir = Paths.get(ProjectContext.DEFAULT_CACHE_DIR); - @CommandLine.Option(defaultValue = "", names={"-p","--projectroot"}, description = "Sets the project root for the current gor query.") - private Path projectRoot; - @CommandLine.Option(names={"-r","--requestid"}, description = "Sets a request id for the current gor query, used to identify logs and errors.") private String requestId = UUID.randomUUID().toString(); @@ -117,7 +114,9 @@ public void run() { // Initialize database connections DbConnection.initInConsoleApp(); - GorExecutionEngine executionEngine = new CLIGorExecutionEngine(commandlineOptions, null, null); + var securityContext = System.getProperty("gor.security.context"); + + GorExecutionEngine executionEngine = new CLIGorExecutionEngine(commandlineOptions, null, securityContext); executionEngine.execute(); } catch (GorException ge) { consoleLogger.error(ExceptionUtilities.gorExceptionToString(ge)); diff --git a/gorscripts/src/test/java/org/gorpipe/gor/cli/UTestGorCliManager.java b/gorscripts/src/test/java/org/gorpipe/gor/cli/UTestGorCliManager.java index 9baf0e2a..33e94895 100644 --- a/gorscripts/src/test/java/org/gorpipe/gor/cli/UTestGorCliManager.java +++ b/gorscripts/src/test/java/org/gorpipe/gor/cli/UTestGorCliManager.java @@ -228,10 +228,10 @@ public void testFlagTagskeyValueCLI() throws Exception { Path dictFile = workDirPath.resolve(name + ".gord"); //Insert 4 files with tags - GorCLI.main(new String[]{"manager", "insert", "--alias", "A", dictFile.toString(), testFiles[0]}); - GorCLI.main(new String[]{"manager", "insert", "--alias", "B", dictFile.toString(), testFiles[1]}); - GorCLI.main(new String[]{"manager", "insert", "--alias", "C", dictFile.toString(), testFiles[2]}); - GorCLI.main(new String[]{"manager", "insert", "--alias", "D", dictFile.toString(), testFiles[3]}); + GorExecCLI.main(new String[]{"manager", "insert", "--alias", "A", dictFile.toString(), testFiles[0]}); + GorExecCLI.main(new String[]{"manager", "insert", "--alias", "B", dictFile.toString(), testFiles[1]}); + GorExecCLI.main(new String[]{"manager", "insert", "--alias", "C", dictFile.toString(), testFiles[2]}); + GorExecCLI.main(new String[]{"manager", "insert", "--alias", "D", dictFile.toString(), testFiles[3]}); TableManager man = new TableManager(); DictionaryTable table = man.initTable(dictFile); @@ -242,14 +242,14 @@ public void testFlagTagskeyValueCLI() throws Exception { //Insert file 1 again but tag it the same as file 4 using the letter "D". //Using the --tagskey flag prevents the same tag to be used twice but instead the new line overwrites the old one. - GorCLI.main(new String[]{"manager", "insert", "--alias", "D", "--tagskey", dictFile.toString(), testFiles[0]}); + GorExecCLI.main(new String[]{"manager", "insert", "--alias", "D", "--tagskey", dictFile.toString(), testFiles[0]}); table.reload(); result = table.selectUninon(table.filter()).stream().map(l -> l.formatEntry()).sorted().collect(Collectors.joining()); Assert.assertEquals("Insert failed", testFiles[0] + "\tA\n" + testFiles[0] + "\tD\n" + testFiles[1] + "\tB\n" + testFiles[2] + "\tC\n", result); //Insert file 1 once again and tag it the same as file 2 using the letter "B" //This should be successful since the --tagskey option is not being used. There should now be two files tagged with "B". - GorCLI.main(new String[]{"manager", "insert", "--alias", "B", dictFile.toString(), testFiles[0]}); + GorExecCLI.main(new String[]{"manager", "insert", "--alias", "B", dictFile.toString(), testFiles[0]}); table.reload(); //Total count of tags Assert.assertEquals(5, table.filter().get().stream().map(l -> l.getFilterTags()).distinct().count()); @@ -266,9 +266,9 @@ public void testFlagTagskeyValueCLI() throws Exception { } //Delete "B" so now there is only one remaining line with the same tag and it should be possible again to update the single existing line. - GorCLI.main(new String[]{"manager", "delete", "--aliases", "B", dictFile.toString(), testFiles[1]}); + GorExecCLI.main(new String[]{"manager", "delete", "--aliases", "B", dictFile.toString(), testFiles[1]}); //Insert tag "B" again which should update the current "B" tagged line with this entry. - GorCLI.main(new String[]{"manager", "insert", "--alias", "B", "--tagskey", dictFile.toString(), testFiles[3]}); + GorExecCLI.main(new String[]{"manager", "insert", "--alias", "B", "--tagskey", dictFile.toString(), testFiles[3]}); table.reload(); result = table.selectUninon(table.filter()).stream().map(l -> l.formatEntry()).sorted().collect(Collectors.joining()); Assert.assertEquals("Insert failed", testFiles[0] + "\tA\n" + testFiles[0] + "\tD\n" + testFiles[2] + "\tC\n" + testFiles[3] + "\tB\n", result); @@ -283,21 +283,21 @@ public void testFlagTagskeyListCLI() throws Exception { DictionaryTable table = man.initTable(dictFile); //Check matching list of tags. If the same set of tags can be found it is replaced by the new line of tags. - GorCLI.main(new String[]{"manager", "insert", "--alias", "A", "--tags", "GO,RC,OR", "--tagskey", dictFile.toString(), testFiles[0]}); - GorCLI.main(new String[]{"manager", "insert", "--alias", "A", "--tags", "GO,RC,OR", "--tagskey", dictFile.toString(), testFiles[1]}); + GorExecCLI.main(new String[]{"manager", "insert", "--alias", "A", "--tags", "GO,RC,OR", "--tagskey", dictFile.toString(), testFiles[0]}); + GorExecCLI.main(new String[]{"manager", "insert", "--alias", "A", "--tags", "GO,RC,OR", "--tagskey", dictFile.toString(), testFiles[1]}); table.reload(); String result = table.selectUninon(table.filter()).stream().map(l -> l.formatEntry()).sorted().collect(Collectors.joining()); Assert.assertEquals("Insert failed", testFiles[1] + "\t\t\t\t\t\tGO,RC,OR,A\n", result); //Check a single tag against a list of tags to confirm that it is not enough to match a single tag in the list but the whole list must match. - GorCLI.main(new String[]{"manager", "insert", "--tags", "GO", "--tagskey", dictFile.toString(), testFiles[1]}); + GorExecCLI.main(new String[]{"manager", "insert", "--tags", "GO", "--tagskey", dictFile.toString(), testFiles[1]}); table.reload(); result = table.selectUninon(table.filter()).stream().map(l -> l.formatEntry()).sorted().collect(Collectors.joining()); Assert.assertEquals("Insert failed", testFiles[1] + "\t\t\t\t\t\tGO,RC,OR,A\n" + testFiles[1] + "\tGO\n", result); //Confirm the other flags such as range are ignored with the --tagskey flag. The second line "updates" the first line with range being ignored and tags only being used as an identifier. - GorCLI.main(new String[]{"manager", "insert", "--alias", "B", "--tags", "GO,RC,OR", "--tagskey", "--range","chr1-chr3", dictFile.toString(), testFiles[2]}); - GorCLI.main(new String[]{"manager", "insert", "--alias", "B", "--tags", "GO,RC,OR", "--tagskey", dictFile.toString(), testFiles[3]}); + GorExecCLI.main(new String[]{"manager", "insert", "--alias", "B", "--tags", "GO,RC,OR", "--tagskey", "--range","chr1-chr3", dictFile.toString(), testFiles[2]}); + GorExecCLI.main(new String[]{"manager", "insert", "--alias", "B", "--tags", "GO,RC,OR", "--tagskey", dictFile.toString(), testFiles[3]}); result = table.selectUninon(table.filter()).stream().map(l -> l.formatEntry()).sorted().collect(Collectors.joining()); table.reload(); Assert.assertEquals("Insert failed", testFiles[1] + "\t\t\t\t\t\tGO,RC,OR,A\n" + testFiles[1] + "\tGO\n" + testFiles[3] + "\t\t\t\t\t\tGO,RC,OR,B\n", result); @@ -308,9 +308,9 @@ public void testDirectCLI() throws Exception { String name = "testDirectCLI"; Path dictFile = workDirPath.resolve(name + ".gord"); - GorCLI.main(new String[]{"manager", "insert", "--alias", "A", dictFile.toString(), testFiles[0]}); - GorCLI.main(new String[]{"manager", "insert", "--alias", "B", dictFile.toString(), testFiles[1]}); - GorCLI.main(new String[]{"manager", "insert", "--alias", "D", dictFile.toString(), testFiles[3]}); + GorExecCLI.main(new String[]{"manager", "insert", "--alias", "A", dictFile.toString(), testFiles[0]}); + GorExecCLI.main(new String[]{"manager", "insert", "--alias", "B", dictFile.toString(), testFiles[1]}); + GorExecCLI.main(new String[]{"manager", "insert", "--alias", "D", dictFile.toString(), testFiles[3]}); TableManager man = new TableManager(); DictionaryTable table = man.initTable(dictFile); @@ -318,12 +318,12 @@ public void testDirectCLI() throws Exception { String result = table.selectUninon(table.filter()).stream().map(l -> l.formatEntry()).sorted().collect(Collectors.joining()); Assert.assertEquals("Insert failed", testFiles[0] + "\tA\n" + testFiles[1] + "\tB\n" + testFiles[3] + "\tD\n", result); - GorCLI.main(new String[]{"manager", "delete", "--tags", "B", dictFile.toString()}); + GorExecCLI.main(new String[]{"manager", "delete", "--tags", "B", dictFile.toString()}); table.reload(); result = table.selectUninon(table.filter().tags("B")).stream().map(l -> l.formatEntry()).sorted().collect(Collectors.joining()); Assert.assertEquals("Delete failed", "", result); - GorCLI.main(new String[]{"manager", "bucketize", "-w", "1", "--min_bucket_size", "1", dictFile.toString()}); + GorExecCLI.main(new String[]{"manager", "bucketize", "-w", "1", "--min_bucket_size", "1", dictFile.toString()}); table.reload(); Assert.assertEquals("Not all lines bucketized", 0, table.needsBucketizing().size()); } diff --git a/gortools/build.gradle b/gortools/build.gradle index b24375b6..cd65ea5d 100644 --- a/gortools/build.gradle +++ b/gortools/build.gradle @@ -58,6 +58,8 @@ project(':gortools') { implementation 'org.apache.parquet:parquet-column:_' implementation 'org.apache.parquet:parquet-hadoop:_' implementation ('org.apache.hadoop:hadoop-common:_') + implementation "info.picocli:picocli:_" + implementation "info.picocli:picocli-shell-jline3:_" runtimeOnly project(':drivers') diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/FormattingOptions.java b/gortools/src/main/java/org/gorpipe/gor/cli/FormattingOptions.java similarity index 100% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/FormattingOptions.java rename to gortools/src/main/java/org/gorpipe/gor/cli/FormattingOptions.java diff --git a/gortools/src/main/java/org/gorpipe/gor/cli/GorExecCLI.java b/gortools/src/main/java/org/gorpipe/gor/cli/GorExecCLI.java new file mode 100644 index 00000000..e68592be --- /dev/null +++ b/gortools/src/main/java/org/gorpipe/gor/cli/GorExecCLI.java @@ -0,0 +1,77 @@ +/* + * BEGIN_COPYRIGHT + * + * Copyright (C) 2011-2013 deCODE genetics Inc. + * Copyright (C) 2013-2019 WuXi NextCode Inc. + * All Rights Reserved. + * + * GORpipe is free software: you can redistribute it and/or modify + * it under the terms of the AFFERO GNU General Public License as published by + * the Free Software Foundation. + * + * GORpipe is distributed "AS-IS" AND WITHOUT ANY WARRANTY OF ANY KIND, + * INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, + * NON-INFRINGEMENT, OR FITNESS FOR A PARTICULAR PURPOSE. See + * the AFFERO GNU General Public License for the complete license terms. + * + * You should have received a copy of the AFFERO GNU General Public License + * along with GORpipe. If not, see + * + * END_COPYRIGHT + */ + +package org.gorpipe.gor.cli; + +import org.gorpipe.gor.cli.help.HelpCommand; +import org.gorpipe.gor.cli.index.IndexCommand; +import org.gorpipe.gor.cli.info.InfoCommand; +import org.gorpipe.gor.cli.link.LinkCommand; +import org.gorpipe.logging.GorLogbackUtil; +import picocli.CommandLine; + +import java.nio.file.Path; + +@SuppressWarnings("squid:S106") +@CommandLine.Command(name="gor", + version="version 1.0", + description = "Command line interface for gor query language and processes.", + subcommands = { + HelpCommand.class, + IndexCommand.class, + InfoCommand.class, + LinkCommand.class}) +public class GorExecCLI extends HelpOptions implements Runnable { + public static void main(String[] args) { + GorLogbackUtil.initLog("gor"); + CommandLine cmd = new CommandLine(new GorExecCLI()); + cmd.parseWithHandlers( + new CommandLine.RunLast(), + CommandLine.defaultExceptionHandler().andExit(-1), + args); + + } + + @CommandLine.Option(names = {"-v", "--version"}, + versionHelp = true, + description = "Print version information and exits.") + boolean versionHelpRequested; + + @CommandLine.Option(defaultValue = "", names={"-p","--projectRoot"}, description = "Sets the project root for the current gor session.") + private Path projectRoot; + + @CommandLine.Option(defaultValue = "", names={"--securityContext"}, description = "Sets the security context for the current gor session.") + private String securityContext; + + @Override + public void run() { + CommandLine.usage(this, System.err); + } + + public String getProjectRoot() { + return projectRoot.toString(); + } + + public String getSecurityContext() { + return securityContext; + } +} diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/HelpOptions.java b/gortools/src/main/java/org/gorpipe/gor/cli/HelpOptions.java similarity index 95% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/HelpOptions.java rename to gortools/src/main/java/org/gorpipe/gor/cli/HelpOptions.java index 775cbc48..8fdc7b4a 100644 --- a/gorscripts/src/main/java/org/gorpipe/gor/cli/HelpOptions.java +++ b/gortools/src/main/java/org/gorpipe/gor/cli/HelpOptions.java @@ -22,6 +22,7 @@ package org.gorpipe.gor.cli; +import org.gorpipe.gor.cli.FormattingOptions; import picocli.CommandLine; public class HelpOptions extends FormattingOptions { diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/cache/AnalysisResult.java b/gortools/src/main/java/org/gorpipe/gor/cli/cache/AnalysisResult.java similarity index 100% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/cache/AnalysisResult.java rename to gortools/src/main/java/org/gorpipe/gor/cli/cache/AnalysisResult.java diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/cache/CacheCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/cache/CacheCommand.java similarity index 100% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/cache/CacheCommand.java rename to gortools/src/main/java/org/gorpipe/gor/cli/cache/CacheCommand.java diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/cache/FilterOptions.java b/gortools/src/main/java/org/gorpipe/gor/cli/cache/FilterOptions.java similarity index 100% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/cache/FilterOptions.java rename to gortools/src/main/java/org/gorpipe/gor/cli/cache/FilterOptions.java diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/cache/PurgeCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/cache/PurgeCommand.java similarity index 100% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/cache/PurgeCommand.java rename to gortools/src/main/java/org/gorpipe/gor/cli/cache/PurgeCommand.java diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/cache/TouchCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/cache/TouchCommand.java similarity index 100% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/cache/TouchCommand.java rename to gortools/src/main/java/org/gorpipe/gor/cli/cache/TouchCommand.java diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/help/HelpCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/help/HelpCommand.java similarity index 97% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/help/HelpCommand.java rename to gortools/src/main/java/org/gorpipe/gor/cli/help/HelpCommand.java index 3ecf4ce7..33a771c4 100644 --- a/gorscripts/src/main/java/org/gorpipe/gor/cli/help/HelpCommand.java +++ b/gortools/src/main/java/org/gorpipe/gor/cli/help/HelpCommand.java @@ -109,7 +109,7 @@ private List loadHelpFiles() throws IOException, URISyntaxException { Map env = new HashMap<>(); env.put("create", "true"); try(FileSystem fileSystem = java.nio.file.FileSystems.newFileSystem(helpJarURL, env)) { - helpList.addAll(java.nio.file.Files.readAllLines(java.nio.file.Paths.get(helpJarURL), java.nio.charset.Charset.forName("ISO-8859-1"))); + helpList.addAll(java.nio.file.Files.readAllLines(java.nio.file.Paths.get(helpJarURL), Charset.forName("ISO-8859-1"))); } } diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/index/IndexCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/index/IndexCommand.java similarity index 100% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/index/IndexCommand.java rename to gortools/src/main/java/org/gorpipe/gor/cli/index/IndexCommand.java diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/info/CommandsCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/info/CommandsCommand.java similarity index 100% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/info/CommandsCommand.java rename to gortools/src/main/java/org/gorpipe/gor/cli/info/CommandsCommand.java diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/info/InfoCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/info/InfoCommand.java similarity index 100% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/info/InfoCommand.java rename to gortools/src/main/java/org/gorpipe/gor/cli/info/InfoCommand.java diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/info/InputsCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/info/InputsCommand.java similarity index 100% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/info/InputsCommand.java rename to gortools/src/main/java/org/gorpipe/gor/cli/info/InputsCommand.java diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/info/MacrosCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/info/MacrosCommand.java similarity index 100% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/info/MacrosCommand.java rename to gortools/src/main/java/org/gorpipe/gor/cli/info/MacrosCommand.java diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/link/LinkCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/link/LinkCommand.java similarity index 65% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/link/LinkCommand.java rename to gortools/src/main/java/org/gorpipe/gor/cli/link/LinkCommand.java index c83be67c..43e1d2aa 100644 --- a/gorscripts/src/main/java/org/gorpipe/gor/cli/link/LinkCommand.java +++ b/gortools/src/main/java/org/gorpipe/gor/cli/link/LinkCommand.java @@ -1,5 +1,6 @@ package org.gorpipe.gor.cli.link; +import org.gorpipe.gor.cli.GorExecCLI; import org.gorpipe.gor.cli.HelpOptions; import picocli.CommandLine; @@ -10,8 +11,19 @@ subcommands = {LinkUpdateCommand.class, LinkRollbackCommand.class, LinkResolveCommand.class}) public class LinkCommand extends HelpOptions implements Runnable { + @CommandLine.ParentCommand + private GorExecCLI gorExecCLI; + @Override public void run() { CommandLine.usage(this, System.err); } + + public String getSecurityContext() { + return gorExecCLI.getSecurityContext(); + } + + public String getProjectRoot() { + return gorExecCLI.getProjectRoot(); + } } diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/link/LinkResolveCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/link/LinkResolveCommand.java similarity index 92% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/link/LinkResolveCommand.java rename to gortools/src/main/java/org/gorpipe/gor/cli/link/LinkResolveCommand.java index 23a22671..99bf2f94 100644 --- a/gorscripts/src/main/java/org/gorpipe/gor/cli/link/LinkResolveCommand.java +++ b/gortools/src/main/java/org/gorpipe/gor/cli/link/LinkResolveCommand.java @@ -1,14 +1,14 @@ package org.gorpipe.gor.cli.link; -import java.io.IOException; -import java.time.Instant; - +import org.gorpipe.gor.cli.GorExecCLI; import org.gorpipe.gor.driver.linkfile.LinkFile; import org.gorpipe.util.DateUtils; import org.gorpipe.util.Strings; - import picocli.CommandLine; +import java.io.IOException; +import java.time.Instant; + @CommandLine.Command(name = "resolve", description = "Resolve a link file to the entry active at the current or given time.") public class LinkResolveCommand implements Runnable { @@ -25,11 +25,14 @@ public class LinkResolveCommand implements Runnable { description = "Return the full link file entry instead of only the resolved URL.") private boolean returnFullEntry; + @CommandLine.ParentCommand + private LinkCommand mainCmd; + @Override public void run() { var normalizedLinkPath = LinkFile.validateAndUpdateLinkFileName(linkFilePath); try { - var linkFile = LinkFile.load(LinkStreamSourceProvider.resolve(normalizedLinkPath, true, this)); + var linkFile = LinkFile.load(LinkStreamSourceProvider.resolve(normalizedLinkPath, mainCmd.getSecurityContext(), mainCmd.getProjectRoot(), true, this)); long timestamp = resolveDate == null ? System.currentTimeMillis() : parseDate(resolveDate); var entry = linkFile.getEntry(timestamp); if (entry == null) { diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/link/LinkRollbackCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/link/LinkRollbackCommand.java similarity index 90% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/link/LinkRollbackCommand.java rename to gortools/src/main/java/org/gorpipe/gor/cli/link/LinkRollbackCommand.java index 4a239fb7..0495cfa3 100644 --- a/gorscripts/src/main/java/org/gorpipe/gor/cli/link/LinkRollbackCommand.java +++ b/gortools/src/main/java/org/gorpipe/gor/cli/link/LinkRollbackCommand.java @@ -1,13 +1,13 @@ package org.gorpipe.gor.cli.link; -import java.io.IOException; -import java.time.Instant; - +import org.gorpipe.gor.cli.GorExecCLI; import org.gorpipe.gor.driver.linkfile.LinkFile; import org.gorpipe.util.DateUtils; - import picocli.CommandLine; +import java.io.IOException; +import java.time.Instant; + @CommandLine.Command(name = "rollback", description = "Rollback the latest entry or rollback entries newer than a given date.") public class LinkRollbackCommand implements Runnable { @@ -19,11 +19,14 @@ public class LinkRollbackCommand implements Runnable { description = "ISO-8601 date/time or epoch milliseconds to rollback to (entries newer than this are removed).") private String rollbackDate; + @CommandLine.ParentCommand + private LinkCommand mainCmd; + @Override public void run() { var normalizedLinkPath = LinkFile.validateAndUpdateLinkFileName(linkFilePath); try { - var linkFile = LinkFile.load(LinkStreamSourceProvider.resolve(normalizedLinkPath, true, this)); + var linkFile = LinkFile.load(LinkStreamSourceProvider.resolve(normalizedLinkPath, mainCmd.getSecurityContext(), mainCmd.getProjectRoot(), true, this)); boolean changed = rollbackDate == null ? linkFile.rollbackLatestEntry() : linkFile.rollbackToTimestamp(parseDate(rollbackDate)); if (!changed) { throw new CommandLine.ParameterException(new CommandLine(this), diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/link/LinkStreamSourceProvider.java b/gortools/src/main/java/org/gorpipe/gor/cli/link/LinkStreamSourceProvider.java similarity index 78% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/link/LinkStreamSourceProvider.java rename to gortools/src/main/java/org/gorpipe/gor/cli/link/LinkStreamSourceProvider.java index bfde54a8..4687500f 100644 --- a/gorscripts/src/main/java/org/gorpipe/gor/cli/link/LinkStreamSourceProvider.java +++ b/gortools/src/main/java/org/gorpipe/gor/cli/link/LinkStreamSourceProvider.java @@ -9,8 +9,8 @@ final class LinkStreamSourceProvider { private LinkStreamSourceProvider() { } - static StreamSource resolve(String linkPath, boolean writeable, Object commandInstance) { - var fileReader = new DriverBackedFileReader("", null); + static StreamSource resolve(String linkPath, String securityContext, String commonRoot, boolean writeable, Object commandInstance) { + var fileReader = new DriverBackedFileReader(securityContext, commonRoot); var dataSource = fileReader.resolveUrl(linkPath, writeable); if (dataSource == null) { throw new CommandLine.ExecutionException(new CommandLine(commandInstance), diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/link/LinkUpdateCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/link/LinkUpdateCommand.java similarity index 90% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/link/LinkUpdateCommand.java rename to gortools/src/main/java/org/gorpipe/gor/cli/link/LinkUpdateCommand.java index c6247e3d..a974b4d3 100644 --- a/gorscripts/src/main/java/org/gorpipe/gor/cli/link/LinkUpdateCommand.java +++ b/gortools/src/main/java/org/gorpipe/gor/cli/link/LinkUpdateCommand.java @@ -1,14 +1,14 @@ package org.gorpipe.gor.cli.link; -import java.io.IOException; -import java.util.LinkedHashMap; -import java.util.Map; - +import org.gorpipe.gor.cli.GorExecCLI; import org.gorpipe.gor.driver.linkfile.LinkFile; import org.gorpipe.util.Strings; - import picocli.CommandLine; +import java.io.IOException; +import java.util.LinkedHashMap; +import java.util.Map; + @CommandLine.Command(name = "update", description = "Append a new entry to a link file, creating the file if needed.") public class LinkUpdateCommand implements Runnable { @@ -33,15 +33,18 @@ public class LinkUpdateCommand implements Runnable { mapFallbackValue = "") private final Map headerParams = new LinkedHashMap<>(); + @CommandLine.ParentCommand + private LinkCommand mainCmd; + @Override public void run() { var normalizedLinkPath = LinkFile.validateAndUpdateLinkFileName(linkFilePath); try { - var linkFile = LinkFile.load(LinkStreamSourceProvider.resolve(normalizedLinkPath, true, this)); + var linkFile = LinkFile.load(LinkStreamSourceProvider.resolve(normalizedLinkPath, mainCmd.getSecurityContext(), mainCmd.getProjectRoot(), true, this)); applyHeaders(linkFile); linkFile.appendEntry(linkValue, entryMd5, entryInfo); linkFile.save(); - System.out.printf("Updated link file %s with %s%n", normalizedLinkPath, linkValue); + System.err.printf("Updated link file %s with %s%n", normalizedLinkPath, linkValue); } catch (IOException e) { throw new CommandLine.ExecutionException(new CommandLine(this), "Failed to load or create link file: " + normalizedLinkPath, e); diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/manager/BucketizeCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/manager/BucketizeCommand.java similarity index 98% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/manager/BucketizeCommand.java rename to gortools/src/main/java/org/gorpipe/gor/cli/manager/BucketizeCommand.java index ec8d2f2c..494fa7f9 100644 --- a/gorscripts/src/main/java/org/gorpipe/gor/cli/manager/BucketizeCommand.java +++ b/gortools/src/main/java/org/gorpipe/gor/cli/manager/BucketizeCommand.java @@ -26,8 +26,6 @@ import org.gorpipe.gor.manager.TableManager; import picocli.CommandLine; -import java.net.URI; -import java.nio.file.Paths; import java.time.Duration; import java.util.ArrayList; import java.util.List; diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/manager/CommandBucketizeOptions.java b/gortools/src/main/java/org/gorpipe/gor/cli/manager/CommandBucketizeOptions.java similarity index 100% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/manager/CommandBucketizeOptions.java rename to gortools/src/main/java/org/gorpipe/gor/cli/manager/CommandBucketizeOptions.java diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/manager/DeleteBucketCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/manager/DeleteBucketCommand.java similarity index 96% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/manager/DeleteBucketCommand.java rename to gortools/src/main/java/org/gorpipe/gor/cli/manager/DeleteBucketCommand.java index 9ac45be3..fbc58992 100644 --- a/gorscripts/src/main/java/org/gorpipe/gor/cli/manager/DeleteBucketCommand.java +++ b/gortools/src/main/java/org/gorpipe/gor/cli/manager/DeleteBucketCommand.java @@ -25,8 +25,6 @@ import org.gorpipe.gor.manager.TableManager; import picocli.CommandLine; -import java.nio.file.Path; -import java.nio.file.Paths; import java.time.Duration; import java.util.ArrayList; import java.util.List; diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/manager/DeleteCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/manager/DeleteCommand.java similarity index 100% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/manager/DeleteCommand.java rename to gortools/src/main/java/org/gorpipe/gor/cli/manager/DeleteCommand.java diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/manager/FilterOptions.java b/gortools/src/main/java/org/gorpipe/gor/cli/manager/FilterOptions.java similarity index 100% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/manager/FilterOptions.java rename to gortools/src/main/java/org/gorpipe/gor/cli/manager/FilterOptions.java diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/manager/InsertCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/manager/InsertCommand.java similarity index 100% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/manager/InsertCommand.java rename to gortools/src/main/java/org/gorpipe/gor/cli/manager/InsertCommand.java diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/manager/ManagerCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/manager/ManagerCommand.java similarity index 100% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/manager/ManagerCommand.java rename to gortools/src/main/java/org/gorpipe/gor/cli/manager/ManagerCommand.java diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/manager/ManagerOptions.java b/gortools/src/main/java/org/gorpipe/gor/cli/manager/ManagerOptions.java similarity index 100% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/manager/ManagerOptions.java rename to gortools/src/main/java/org/gorpipe/gor/cli/manager/ManagerOptions.java diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/manager/MultiInsertCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/manager/MultiInsertCommand.java similarity index 100% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/manager/MultiInsertCommand.java rename to gortools/src/main/java/org/gorpipe/gor/cli/manager/MultiInsertCommand.java diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/manager/SelectCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/manager/SelectCommand.java similarity index 100% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/manager/SelectCommand.java rename to gortools/src/main/java/org/gorpipe/gor/cli/manager/SelectCommand.java diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/manager/TestCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/manager/TestCommand.java similarity index 100% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/manager/TestCommand.java rename to gortools/src/main/java/org/gorpipe/gor/cli/manager/TestCommand.java diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/manager/TestIsLockCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/manager/TestIsLockCommand.java similarity index 100% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/manager/TestIsLockCommand.java rename to gortools/src/main/java/org/gorpipe/gor/cli/manager/TestIsLockCommand.java diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/manager/TestReadLockCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/manager/TestReadLockCommand.java similarity index 100% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/manager/TestReadLockCommand.java rename to gortools/src/main/java/org/gorpipe/gor/cli/manager/TestReadLockCommand.java diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/manager/TestWriteLockCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/manager/TestWriteLockCommand.java similarity index 100% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/manager/TestWriteLockCommand.java rename to gortools/src/main/java/org/gorpipe/gor/cli/manager/TestWriteLockCommand.java diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/render/RenderCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/render/RenderCommand.java similarity index 100% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/render/RenderCommand.java rename to gortools/src/main/java/org/gorpipe/gor/cli/render/RenderCommand.java diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/render/RenderOptions.java b/gortools/src/main/java/org/gorpipe/gor/cli/render/RenderOptions.java similarity index 100% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/render/RenderOptions.java rename to gortools/src/main/java/org/gorpipe/gor/cli/render/RenderOptions.java diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/render/ReportCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/render/ReportCommand.java similarity index 100% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/render/ReportCommand.java rename to gortools/src/main/java/org/gorpipe/gor/cli/render/ReportCommand.java diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/render/ScriptCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/render/ScriptCommand.java similarity index 100% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/render/ScriptCommand.java rename to gortools/src/main/java/org/gorpipe/gor/cli/render/ScriptCommand.java diff --git a/gortools/src/main/scala/gorsat/InputSources/Exec.scala b/gortools/src/main/scala/gorsat/InputSources/Exec.scala new file mode 100644 index 00000000..c330864c --- /dev/null +++ b/gortools/src/main/scala/gorsat/InputSources/Exec.scala @@ -0,0 +1,100 @@ +/* + * BEGIN_COPYRIGHT + * + * Copyright (C) 2011-2013 deCODE genetics Inc. + * Copyright (C) 2013-2019 WuXi NextCode Inc. + * All Rights Reserved. + * + * GORpipe is free software: you can redistribute it and/or modify + * it under the terms of the AFFERO GNU General Public License as published by + * the Free Software Foundation. + * + * GORpipe is distributed "AS-IS" AND WITHOUT ANY WARRANTY OF ANY KIND, + * INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, + * NON-INFRINGEMENT, OR FITNESS FOR A PARTICULAR PURPOSE. See + * the AFFERO GNU General Public License for the complete license terms. + * + * You should have received a copy of the AFFERO GNU General Public License + * along with GORpipe. If not, see + * + * END_COPYRIGHT + */ + +package gorsat.InputSources + +import gorsat.Commands.{CommandArguments, CommandParseUtilities, InputSourceInfo, InputSourceParsingResult} +import gorsat.Iterators.{CountingNorRowIterator, RowListIterator} +import org.gorpipe.gor.cli.GorExecCLI +import org.gorpipe.gor.cli.link.LinkUpdateCommand +import org.gorpipe.gor.model.{GorCommand, NoValidateRowBase, Row} +import org.gorpipe.gor.session.GorContext +import picocli.CommandLine + +import java.io.{ByteArrayOutputStream, PrintStream, PrintWriter} +import scala.collection.mutable.ListBuffer +import scala.util.Using + +/** + * Execute selected gor commands in NOR context. + */ +class Exec() extends InputSourceInfo("EXEC", CommandArguments("","", 2, 10, ignoreIllegalArguments=true), isNorCommand = true) { + + override def processArguments(context: GorContext, argString: String, iargs: Array[String], + args: Array[String]): InputSourceParsingResult = { + + val result = processExec(context, argString, iargs.slice(1, iargs.length), args) + + val header = "Chrom\tpos\t" + result(0) + val myHeaderLength = header.split("\t").length + val lineList = new ListBuffer[Row]() + for (row <- result.slice(1, result.length)) { + lineList += new NoValidateRowBase("chrN\t0\t" + row, myHeaderLength) + } + + val inputSource = RowListIterator(lineList.toList) + inputSource.setHeader(header) + InputSourceParsingResult(inputSource, header, isNorContext = true) + } + + def processExec(context: GorContext, argString: String, iargs: Array[String], + args: Array[String]): Array[String] = { + + + val oldOut = System.out + val oldErr = System.err + val std_baos = new ByteArrayOutputStream() + val err_baos = new ByteArrayOutputStream() + val std_ps = new PrintStream(std_baos, true) + val err_ps = new PrintStream(std_baos, true) + var exitCode = -128 + + try { + System.setOut(std_ps) + System.setErr(err_ps) + exitCode = new CommandLine(new GorExecCLI) + .setExitCodeExceptionMapper(new CommandLine.IExitCodeExceptionMapper { + override def getExitCode(e: Throwable): Int = { + // Don't map exist codes. + throw new IllegalArgumentException(s"EXEC command: ${argString} failed: ${e.getMessage}", e) + } + }) + .setDefaultValueProvider((argSpec: CommandLine.Model.ArgSpec) => { + if (argSpec.paramLabel() == "") { + context.getSession.getProjectContext.getFileReader.getSecurityContext + } else if (argSpec.paramLabel() == "") { + context.getSession.getProjectContext.getFileReader.getCommonRoot + } else { + null + } + }) + .execute(args.slice(1, args.length): _*) + } finally { + System.setOut(oldOut) + System.setErr(oldErr) + std_ps.close() + err_ps.close() + } + + Array("Status\tStdOut\tStdErr", exitCode + "\t\"" + std_baos.toString.stripLineEnd + "\"\t\"" + err_baos.toString.stripLineEnd + "\"") + } +} diff --git a/gortools/src/main/scala/gorsat/process/GorInputSources.scala b/gortools/src/main/scala/gorsat/process/GorInputSources.scala index 9cee08ee..b3396f58 100644 --- a/gortools/src/main/scala/gorsat/process/GorInputSources.scala +++ b/gortools/src/main/scala/gorsat/process/GorInputSources.scala @@ -23,7 +23,7 @@ package gorsat.process import gorsat.Commands._ -import gorsat.InputSources.{Cmd, Gorif, Meta, Nor, Sql} +import gorsat.InputSources.{Cmd, Gorif, Meta, Nor, Sql, Exec} /** * Methods to register and access gor input sources. Input sources need to be registered before use. @@ -80,6 +80,7 @@ object GorInputSources { addInfo(new Sql.NorSql) addInfo(new Sql.GorSql) addInfo(new Meta) + addInfo(new Exec) } } diff --git a/gortools/src/test/java/gorsat/Inputs/UTestLink.java b/gortools/src/test/java/gorsat/Inputs/UTestLink.java new file mode 100644 index 00000000..3f0b430d --- /dev/null +++ b/gortools/src/test/java/gorsat/Inputs/UTestLink.java @@ -0,0 +1,164 @@ +package gorsat.Inputs; + +import gorsat.Commands.CommandParseUtilities; +import gorsat.TestUtils; +import org.gorpipe.gor.driver.linkfile.LinkFile; +import org.gorpipe.gor.driver.providers.stream.sources.file.FileSource; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.nio.file.Path; +import java.time.Instant; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class UTestLink { + @Rule + public TemporaryFolder temp = new TemporaryFolder(); + + @Test + public void testLinkHelp() throws Exception { + String res = TestUtils.runGorPipe("exec gor link --help "); + + assertEquals("0", res.split("\n")[1].split("\t")[2]); + assertTrue(res.split("\n")[1].split("\t")[3].startsWith("\"Link file management commands.")); + } + + @Test + public void testUpdateCreatesLinkFileError() throws Exception { + Path linkFile = temp.getRoot().toPath().resolve("update_test.gor.link"); + + String res = TestUtils.runGorPipe("exec gor link update " + linkFile.toString() + " data/file1.gor -xxx ENTRIES_COUNT_MAX=5"); + + LinkFile link = LinkFile.load(new FileSource(linkFile)); + assertEquals(0, link.getEntriesCount()); + assertEquals("2", res.split("\n")[1].split("\t")[2]); + assertEquals("\"Unknown options: '-xxx', 'ENTRIES_COUNT_MAX=5'", res.split("\n")[1].split("\t")[3]); + } + + @Test + public void testUpdateCreatesLinkFileAndAppliesHeaders() throws Exception { + Path linkFile = temp.getRoot().toPath().resolve("update_test.gor.link"); + + String res = TestUtils.runGorPipe("exec gor link update " + linkFile.toString() + " data/file1.gor -h ENTRIES_COUNT_MAX=5"); + + LinkFile link = LinkFile.load(new FileSource(linkFile)); + assertEquals("0", res.split("\n")[1].split("\t")[2]); + assertEquals(1, link.getEntriesCount()); + assertEquals(resolve(linkFile, "data/file1.gor"), link.getLatestEntryUrl()); + assertEquals(5, link.getEntriesCountMax()); + } + + @Test + public void testUpdateCreatesLinkFileWithProjectRooot() throws Exception { + Path linkFile = temp.getRoot().toPath().resolve("update_test.gor.link"); + + String res = TestUtils.runGorPipe("exec gor link update update_test.gor.link data/file1.gor -h ENTRIES_COUNT_MAX=5", temp.getRoot().toPath().toString() , true, "dummy"); + + LinkFile link = LinkFile.load(new FileSource(linkFile)); + assertEquals("0", res.split("\n")[1].split("\t")[2]); + assertEquals(1, link.getEntriesCount()); + assertEquals(resolve(linkFile, "data/file1.gor"), link.getLatestEntryUrl()); + assertEquals(5, link.getEntriesCountMax()); + } + + @Test + public void testUpdateWithMd5AndInfo() throws Exception { + Path linkFile = temp.getRoot().toPath().resolve("update_md5_info.gor.link"); + + String res = TestUtils.runGorPipe("exec gor link update " + linkFile.toString() + " data/file1.gor -m abc123 -i 'first entry'"); + + LinkFile link = LinkFile.load(new FileSource(linkFile)); + var latest = link.getLatestEntry(); + assertEquals("0", res.split("\n")[1].split("\t")[2]); + assertEquals("abc123", latest.md5()); + assertEquals("'first entry'", latest.info()); + } + + @Test + public void testRollbackLatestEntry() throws Exception { + Path linkFile = temp.getRoot().toPath().resolve("rollback_latest.gor.link"); + + TestUtils.runGorPipe("exec gor link update " + linkFile.toString() + " data/file1.gor"); + Thread.sleep(5); + TestUtils.runGorPipe("exec gor link update " + linkFile.toString() + " data/file2.gor"); + + String res = TestUtils.runGorPipe("exec gor link rollback " + linkFile.toString()); + + assertEquals("0", res.split("\n")[1].split("\t")[2]); + LinkFile link = LinkFile.load(new FileSource(linkFile)); + assertEquals(1, link.getEntriesCount()); + assertEquals(resolve(linkFile, "data/file1.gor"), link.getLatestEntryUrl()); + } + + @Test + public void testRollbackToDate() throws Exception { + Path linkFile = temp.getRoot().toPath().resolve("rollback_date.gor.link"); + + TestUtils.runGorPipe("exec gor link update " + linkFile.toString() + " data/file1.gor"); + LinkFile first = LinkFile.load(new FileSource(linkFile)); + long firstTimestamp = first.getLatestEntry().timestamp(); + Thread.sleep(5); + TestUtils.runGorPipe("exec gor link update " + linkFile.toString() + " data/file2.gor"); + + String rollbackIso = Instant.ofEpochMilli(firstTimestamp).toString(); + String res = TestUtils.runGorPipe("exec gor link rollback " + linkFile.toString() + " -d " + rollbackIso); + + assertEquals("0", res.split("\n")[1].split("\t")[2]); + LinkFile link = LinkFile.load(new FileSource(linkFile)); + assertEquals(1, link.getEntriesCount()); + assertEquals(resolve(linkFile, "data/file1.gor"), link.getLatestEntryUrl()); + } + + @Test + public void testResolveLatestEntry() throws Exception { + Path linkFile = temp.getRoot().toPath().resolve("resolve_latest.gor.link"); + + TestUtils.runGorPipe("exec gor link update " + linkFile.toString() + " data/file1.gor"); + Thread.sleep(5); + TestUtils.runGorPipe("exec gor link update " + linkFile.toString() + " data/file2.gor"); + + String res = TestUtils.runGorPipe("exec gor link resolve " + linkFile); + + assertEquals("0", res.split("\n")[1].split("\t")[2]); + assertEquals("\"" + resolve(linkFile, "data/file2.gor") + "\"", res.split("\n")[1].split("\t")[3]); + } + + @Test + public void testResolveSpecificDate() throws Exception { + Path linkFile = temp.getRoot().toPath().resolve("resolve_date.gor.link"); + + TestUtils.runGorPipe("exec gor link update " + linkFile.toString() + " data/file1.gor"); + LinkFile first = LinkFile.load(new FileSource(linkFile)); + long firstTimestamp = first.getLatestEntry().timestamp(); + Thread.sleep(5); + TestUtils.runGorPipe("exec gor link update " + linkFile.toString() + " data/file2.gor"); + + String res = TestUtils.runGorPipe("exec gor link resolve " + linkFile.toString() + " -d " + Instant.ofEpochMilli(firstTimestamp).toString()); + + assertEquals("0", res.split("\n")[1].split("\t")[2]); + assertEquals("\"" + resolve(linkFile, "data/file1.gor") + "\"", res.split("\n")[1].split("\t")[3]); + } + + @Test + public void testResolveFullEntry() throws Exception { + Path linkFile = temp.getRoot().toPath().resolve("resolve_full.gor.link"); + + TestUtils.runGorPipe("exec gor link update " + linkFile.toString() + " data/file1.gor"); + Thread.sleep(5); + TestUtils.runGorPipe("exec gor link update " + linkFile.toString() + " data/file2.gor"); + + String expectedEntry = LinkFile.load(new FileSource(linkFile)).getLatestEntry().format(); + + String res = TestUtils.runGorPipe("exec gor link resolve " + linkFile.toString() + " -f"); + + assertEquals("0", res.split("\n")[1].split("\t")[2]); + assertEquals("\"" + expectedEntry + "\"", CommandParseUtilities.quoteSafeSplit(res.split("\n")[1], '\t')[3]); + } + + private String resolve(Path linkFile, String relative) { + return linkFile.getParent().resolve(relative).toAbsolutePath().normalize().toString(); + } +} From 4cc0484f0d86854dc5bc2a06e227c088c057f0f7 Mon Sep 17 00:00:00 2001 From: Gisli Magnusson Date: Mon, 1 Dec 2025 21:48:21 -0500 Subject: [PATCH 05/17] feat(ENGKNOW-2892): Add gor query support for link files in gor server. --- .../java/org/gorpipe/gor/cli/query/QueryCommand.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/query/QueryCommand.java b/gorscripts/src/main/java/org/gorpipe/gor/cli/query/QueryCommand.java index a53d6553..2c8554f6 100644 --- a/gorscripts/src/main/java/org/gorpipe/gor/cli/query/QueryCommand.java +++ b/gorscripts/src/main/java/org/gorpipe/gor/cli/query/QueryCommand.java @@ -28,6 +28,8 @@ import gorsat.process.PipeOptions; import org.gorpipe.exceptions.ExceptionUtilities; import org.gorpipe.exceptions.GorException; +import org.gorpipe.gor.cli.GorCLI; +import org.gorpipe.gor.cli.GorExecCLI; import org.gorpipe.gor.cli.HelpOptions; import org.gorpipe.gor.model.DbConnection; import org.gorpipe.gor.session.ProjectContext; @@ -78,6 +80,9 @@ public class QueryCommand extends HelpOptions implements Runnable{ @CommandLine.Parameters(index = "0", arity = "1", paramLabel = "InputQuery", description = "Queries to execute. Queries can be direct gor query, files containing gor script or gor report template.") private String query; + @CommandLine.ParentCommand + private GorCLI parentCommand; + @Override public void run() { @@ -92,8 +97,8 @@ public void run() { commandlineOptions.aliasFile_$eq(aliasFile.toString()); if (configFile != null) commandlineOptions.configFile_$eq(configFile.toString()); - if (projectRoot != null) - commandlineOptions.gorRoot_$eq(projectRoot.toString()); + if (parentCommand.getProjectRoot() != null) + commandlineOptions.gorRoot_$eq(parentCommand.getProjectRoot().toString()); commandlineOptions.requestId_$eq(requestId); commandlineOptions.showStackTrace_$eq(showStackTrace); commandlineOptions.workers_$eq(workers); From aa27da24b00eb438b24bff7eb61709c04e8adeda Mon Sep 17 00:00:00 2001 From: Gisli Magnusson Date: Mon, 1 Dec 2025 22:41:45 -0500 Subject: [PATCH 06/17] feat(ENGKNOW-2892): Add gor query support for link files in gor server. --- gorscripts/build.gradle | 1 - .../main/java/org/gorpipe/gor/cli/GorCLI.java | 11 +-- gortools/build.gradle | 1 + .../org/gorpipe/gor/cli/GorConfigDoc.java | 0 .../java/org/gorpipe/gor/cli/GorExecCLI.java | 12 ++- .../gor/cli/git/GitCheckoutCommand.java | 77 +++++++++++++++ .../gorpipe/gor/cli/git/GitCloneCommand.java | 84 ++++++++++++++++ .../org/gorpipe/gor/cli/git/GitCommand.java | 61 ++++++++++++ .../gor/cli/git/GitCommandExecutor.java | 79 +++++++++++++++ .../gorpipe/gor/cli/git/GitCommitCommand.java | 89 +++++++++++++++++ .../gorpipe/gor/cli/git/GitPullCommand.java | 96 +++++++++++++++++++ .../gorpipe/gor/cli/git/GitPushCommand.java | 96 +++++++++++++++++++ .../org/gorpipe/gor/cli/link/LinkCommand.java | 6 +- .../gor/cli/link/LinkResolveCommand.java | 1 - .../gor/cli/link/LinkRollbackCommand.java | 1 - .../gor/cli/link/LinkUpdateCommand.java | 1 - .../main/scala/gorsat/InputSources/Exec.scala | 10 +- 17 files changed, 599 insertions(+), 27 deletions(-) rename {gorscripts => gortools}/src/main/java/org/gorpipe/gor/cli/GorConfigDoc.java (100%) create mode 100644 gortools/src/main/java/org/gorpipe/gor/cli/git/GitCheckoutCommand.java create mode 100644 gortools/src/main/java/org/gorpipe/gor/cli/git/GitCloneCommand.java create mode 100644 gortools/src/main/java/org/gorpipe/gor/cli/git/GitCommand.java create mode 100644 gortools/src/main/java/org/gorpipe/gor/cli/git/GitCommandExecutor.java create mode 100644 gortools/src/main/java/org/gorpipe/gor/cli/git/GitCommitCommand.java create mode 100644 gortools/src/main/java/org/gorpipe/gor/cli/git/GitPullCommand.java create mode 100644 gortools/src/main/java/org/gorpipe/gor/cli/git/GitPushCommand.java diff --git a/gorscripts/build.gradle b/gorscripts/build.gradle index b0ef52b5..673aa4a3 100644 --- a/gorscripts/build.gradle +++ b/gorscripts/build.gradle @@ -48,7 +48,6 @@ project(':gorscripts') { implementation "org.apache.commons:commons-lang3:_" implementation "de.tototec:de.tototec.cmdoption:_" implementation "com.fasterxml.jackson.core:jackson-databind:_" - implementation "org.reflections:reflections:_" implementation "com.google.guava:guava:_" runtimeOnly project(':documentation') diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/GorCLI.java b/gorscripts/src/main/java/org/gorpipe/gor/cli/GorCLI.java index 314f2ec1..f99e3a97 100644 --- a/gorscripts/src/main/java/org/gorpipe/gor/cli/GorCLI.java +++ b/gorscripts/src/main/java/org/gorpipe/gor/cli/GorCLI.java @@ -38,13 +38,11 @@ @CommandLine.Command(name="gor", version="version 1.0", description = "Command line interface for gor query language and processes.", - subcommands = {QueryCommand.class, HelpCommand.class, ManagerCommand.class, IndexCommand.class, - CacheCommand.class, RenderCommand.class, InfoCommand.class, FolderMigratorCommand.class, - LinkCommand.class}) + subcommands = {QueryCommand.class, FolderMigratorCommand.class}) public class GorCLI extends GorExecCLI implements Runnable { public static void main(String[] args) { GorLogbackUtil.initLog("gor"); - CommandLine cmd = new CommandLine(new GorExecCLI()); + CommandLine cmd = new CommandLine(new GorCLI()); cmd.parseWithHandlers( new CommandLine.RunLast(), CommandLine.defaultExceptionHandler().andExit(-1), @@ -52,11 +50,6 @@ public static void main(String[] args) { } - @CommandLine.Option(names = {"-v", "--version"}, - versionHelp = true, - description = "Print version information and exits.") - boolean versionHelpRequested; - @Override public void run() { CommandLine.usage(this, System.err); diff --git a/gortools/build.gradle b/gortools/build.gradle index cd65ea5d..a8bb4c0d 100644 --- a/gortools/build.gradle +++ b/gortools/build.gradle @@ -60,6 +60,7 @@ project(':gortools') { implementation ('org.apache.hadoop:hadoop-common:_') implementation "info.picocli:picocli:_" implementation "info.picocli:picocli-shell-jline3:_" + implementation "org.reflections:reflections:_" runtimeOnly project(':drivers') diff --git a/gorscripts/src/main/java/org/gorpipe/gor/cli/GorConfigDoc.java b/gortools/src/main/java/org/gorpipe/gor/cli/GorConfigDoc.java similarity index 100% rename from gorscripts/src/main/java/org/gorpipe/gor/cli/GorConfigDoc.java rename to gortools/src/main/java/org/gorpipe/gor/cli/GorConfigDoc.java diff --git a/gortools/src/main/java/org/gorpipe/gor/cli/GorExecCLI.java b/gortools/src/main/java/org/gorpipe/gor/cli/GorExecCLI.java index e68592be..a5fb0f8d 100644 --- a/gortools/src/main/java/org/gorpipe/gor/cli/GorExecCLI.java +++ b/gortools/src/main/java/org/gorpipe/gor/cli/GorExecCLI.java @@ -22,10 +22,14 @@ package org.gorpipe.gor.cli; +import org.gorpipe.gor.cli.cache.CacheCommand; +import org.gorpipe.gor.cli.git.GitCommand; import org.gorpipe.gor.cli.help.HelpCommand; import org.gorpipe.gor.cli.index.IndexCommand; import org.gorpipe.gor.cli.info.InfoCommand; import org.gorpipe.gor.cli.link.LinkCommand; +import org.gorpipe.gor.cli.manager.ManagerCommand; +import org.gorpipe.gor.cli.render.RenderCommand; import org.gorpipe.logging.GorLogbackUtil; import picocli.CommandLine; @@ -35,11 +39,9 @@ @CommandLine.Command(name="gor", version="version 1.0", description = "Command line interface for gor query language and processes.", - subcommands = { - HelpCommand.class, - IndexCommand.class, - InfoCommand.class, - LinkCommand.class}) + subcommands = {HelpCommand.class, ManagerCommand.class, IndexCommand.class, + CacheCommand.class, RenderCommand.class, InfoCommand.class, + LinkCommand.class, GitCommand.class}) public class GorExecCLI extends HelpOptions implements Runnable { public static void main(String[] args) { GorLogbackUtil.initLog("gor"); diff --git a/gortools/src/main/java/org/gorpipe/gor/cli/git/GitCheckoutCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/git/GitCheckoutCommand.java new file mode 100644 index 00000000..32a5accf --- /dev/null +++ b/gortools/src/main/java/org/gorpipe/gor/cli/git/GitCheckoutCommand.java @@ -0,0 +1,77 @@ +package org.gorpipe.gor.cli.git; + +import picocli.CommandLine; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; + +@CommandLine.Command(name = "checkout", + description = "Switch branches or restore working tree files.") +public class GitCheckoutCommand implements Runnable { + + @CommandLine.Parameters(index = "0", arity = "1..*", paramLabel = "BRANCH_OR_FILE", + description = "Branch, commit or files to checkout.") + private List branchOrFiles; + + @CommandLine.Option(names = {"-d", "--directory"}, paramLabel = "DIRECTORY", + description = "Directory to work in.", defaultValue = ".") + private String directory; + + @CommandLine.Option(names = {"-b", "--branch"}, paramLabel = "NEW_BRANCH", + description = "Create a new branch and switch to it.") + private String newBranch; + + @CommandLine.Option(names = {"-B", "--force-branch"}, paramLabel = "NEW_BRANCH", + description = "Create a new branch or reset an existing branch and switch to it.") + private String forceBranch; + + @CommandLine.Option(names = {"-f", "--force"}, + description = "Force checkout (throw away local changes).") + private boolean force; + + @CommandLine.Option(names = {"-q", "--quiet"}, + description = "Suppress feedback messages.") + private boolean quiet; + + @CommandLine.ParentCommand + private GitCommand parentCommand; + + @CommandLine.Spec + private CommandLine.Model.CommandSpec spec; + + @Override + public void run() { + if (branchOrFiles == null || branchOrFiles.isEmpty()) { + throw new CommandLine.ParameterException(spec.commandLine(), + "At least one branch or file must be specified."); + } + + List args = new ArrayList<>(); + + if (newBranch != null) { + args.add("-b"); + args.add(newBranch); + } + + if (forceBranch != null) { + args.add("-B"); + args.add(forceBranch); + } + + if (force) { + args.add("-f"); + } + + if (quiet) { + args.add("-q"); + } + + args.addAll(branchOrFiles); + + File workingDir = parentCommand.getWorkingDirectory(directory); + + GitCommandExecutor.executeGitCommand("checkout", args, workingDir, spec); + } +} + diff --git a/gortools/src/main/java/org/gorpipe/gor/cli/git/GitCloneCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/git/GitCloneCommand.java new file mode 100644 index 00000000..821d0794 --- /dev/null +++ b/gortools/src/main/java/org/gorpipe/gor/cli/git/GitCloneCommand.java @@ -0,0 +1,84 @@ +package org.gorpipe.gor.cli.git; + +import picocli.CommandLine; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; + +@CommandLine.Command(name = "clone", + description = "Clone a repository into a new directory.") +public class GitCloneCommand implements Runnable { + + @CommandLine.Parameters(index = "0", paramLabel = "REPOSITORY", + description = "The repository to clone from, just the repository name, e.g. ref-gregor.") + private String repository; + + @CommandLine.Parameters(index = "1", arity = "0..1", paramLabel = "DIRECTORY", + description = "The name of a new directory to clone into.") + private String directory; + + @CommandLine.Option(names = {"--depth"}, paramLabel = "DEPTH", + description = "Create a shallow clone with a history truncated to the specified number of commits.") + private Integer depth; + + @CommandLine.Option(names = {"-b", "--branch"}, paramLabel = "BRANCH", + description = "Clone a specific branch instead of the branch pointed to by HEAD.") + private String branch; + + @CommandLine.Option(names = {"--single-branch"}, + description = "Clone only the history leading to the tip of a single branch.") + private boolean singleBranch; + + @CommandLine.Option(names = {"--no-checkout"}, + description = "Do not checkout HEAD after cloning is complete.") + private boolean noCheckout; + + @CommandLine.Option(names = {"--bare"}, + description = "Make a bare Git repository.") + private boolean bare; + + @CommandLine.ParentCommand + private GitCommand parentCommand; + + @CommandLine.Spec + private CommandLine.Model.CommandSpec spec; + + @Override + public void run() { + List args = new ArrayList<>(); + + if (depth != null) { + args.add("--depth"); + args.add(depth.toString()); + } + + if (branch != null) { + args.add("-b"); + args.add(branch); + } + + if (singleBranch) { + args.add("--single-branch"); + } + + if (noCheckout) { + args.add("--no-checkout"); + } + + if (bare) { + args.add("--bare"); + } + + args.add(parentCommand.getFullRepositoryPath(repository)); + + if (directory != null) { + args.add(directory); + } + + File workingDir = parentCommand.getWorkingDirectory(directory); + + GitCommandExecutor.executeGitCommand("clone", args, workingDir, spec); + } +} + diff --git a/gortools/src/main/java/org/gorpipe/gor/cli/git/GitCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/git/GitCommand.java new file mode 100644 index 00000000..348c1447 --- /dev/null +++ b/gortools/src/main/java/org/gorpipe/gor/cli/git/GitCommand.java @@ -0,0 +1,61 @@ +package org.gorpipe.gor.cli.git; + +import org.gorpipe.gor.cli.GorExecCLI; +import org.gorpipe.gor.cli.HelpOptions; +import org.gorpipe.gor.table.util.PathUtils; +import org.gorpipe.util.Strings; +import picocli.CommandLine; + +import java.io.File; + +@SuppressWarnings("squid:S106") +@CommandLine.Command(name = "git", + description = "Wrapper for git commands (clone, checkout, pull, push, commit).", + header = "Git command wrapper.", + subcommands = {GitCloneCommand.class, GitCheckoutCommand.class, GitPullCommand.class, GitPushCommand.class, GitCommitCommand.class}) +public class GitCommand extends HelpOptions implements Runnable { + + @CommandLine.ParentCommand + private GorExecCLI parentCommand; + + @Override + public void run() { + CommandLine.usage(this, System.err); + } + + public String getSecurityContext() { + return parentCommand != null ? parentCommand.getSecurityContext() : ""; + } + + public String getProjectRoot() { + return parentCommand != null ? parentCommand.getProjectRoot() : ""; + } + + public String getFullRepositoryPath(String repository) { + if (repository.equals("origin")) { + return repository; + } + String user = System.getenv("GOR_GIT_USER"); + String pass = System.getenv("GOR_GIT_PASS"); + if (user != null && pass != null) { + var userPass = "%s:%s@".formatted(user, pass); + return "https://%sgithub.com/GeneDx/%s.git".formatted(userPass, repository); + } else { + return "git@github.com:GeneDx/%s.git".formatted(repository); + } + } + + public File getWorkingDirectory(String directory) { + File workingDir = null; + if (!Strings.isNullOrEmpty(directory)) { + + if (!Strings.isNullOrEmpty(parentCommand.getProjectRoot())) { + workingDir = new File(PathUtils.resolve(parentCommand.getProjectRoot(), directory).toString()); + } else { + workingDir = new File(directory); + } + } + return workingDir; + } +} + diff --git a/gortools/src/main/java/org/gorpipe/gor/cli/git/GitCommandExecutor.java b/gortools/src/main/java/org/gorpipe/gor/cli/git/GitCommandExecutor.java new file mode 100644 index 00000000..4a8a1c04 --- /dev/null +++ b/gortools/src/main/java/org/gorpipe/gor/cli/git/GitCommandExecutor.java @@ -0,0 +1,79 @@ +package org.gorpipe.gor.cli.git; + +import picocli.CommandLine; + +import java.io.BufferedReader; +import java.io.File; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; + +/** + * Utility class for executing git commands. + */ +class GitCommandExecutor { + + /** + * Execute a git command with the given arguments. + * + * @param gitSubcommand the git subcommand (e.g., "clone", "checkout", "pull", "push") + * @param args additional arguments to pass to git + * @param workingDir the working directory for the command (null for current directory) + * @param commandSpec the CommandLine spec for error reporting + * @return the exit code of the git command + */ + static int executeGitCommand(String gitSubcommand, List args, File workingDir, + CommandLine.Model.CommandSpec commandSpec) { + List command = new ArrayList<>(); + command.add("git"); + command.add(gitSubcommand); + command.addAll(args); + + try { + ProcessBuilder pb = new ProcessBuilder(command); + if (workingDir != null) { + pb.directory(workingDir); + } + pb.redirectErrorStream(false); + + Process process = pb.start(); + + // Stream stdout to System.out + streamOutput(process.getInputStream(), System.out); + + // Stream stderr to System.err + streamOutput(process.getErrorStream(), System.err); + + int exitCode = process.waitFor(); + + if (exitCode != 0) { + throw new CommandLine.ExecutionException(commandSpec.commandLine(), + String.format("Git command '%s' failed with exit code %d", gitSubcommand, exitCode)); + } + + return exitCode; + } catch (CommandLine.ExecutionException e) { + throw e; + } catch (Exception e) { + throw new CommandLine.ExecutionException(commandSpec.commandLine(), + String.format("Failed to execute git command '%s': %s", gitSubcommand, e.getMessage()), e); + } + } + + private static void streamOutput(InputStream inputStream, java.io.PrintStream outputStream) { + new Thread(() -> { + try (BufferedReader reader = new BufferedReader( + new InputStreamReader(inputStream, StandardCharsets.UTF_8))) { + String line; + while ((line = reader.readLine()) != null) { + outputStream.println(line); + } + } catch (Exception e) { + // Ignore errors in output streaming + } + }).start(); + } +} + diff --git a/gortools/src/main/java/org/gorpipe/gor/cli/git/GitCommitCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/git/GitCommitCommand.java new file mode 100644 index 00000000..d255a322 --- /dev/null +++ b/gortools/src/main/java/org/gorpipe/gor/cli/git/GitCommitCommand.java @@ -0,0 +1,89 @@ +package org.gorpipe.gor.cli.git; + +import picocli.CommandLine; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; + +@CommandLine.Command(name = "commit", + description = "Record changes to the repository.") +public class GitCommitCommand implements Runnable { + + @CommandLine.Option(names = {"-m", "--message"}, paramLabel = "MESSAGE", + description = "Use the given message as the commit message.") + private String message; + + @CommandLine.Option(names = {"-a", "--all"}, + description = "Automatically stage files that have been modified and deleted.") + private boolean all; + + @CommandLine.Option(names = {"--amend"}, + description = "Amend the previous commit.") + private boolean amend; + + @CommandLine.Option(names = {"--no-verify", "-n"}, + description = "Bypass pre-commit and commit-msg hooks.") + private boolean noVerify; + + @CommandLine.Option(names = {"-v", "--verbose"}, + description = "Show unified diff of all file changes in the commit message template.") + private boolean verbose; + + @CommandLine.Option(names = {"-q", "--quiet"}, + description = "Suppress commit summary message.") + private boolean quiet; + + @CommandLine.Option(names = {"-d", "--directory"}, paramLabel = "DIRECTORY", + description = "Directory to work in.", defaultValue = ".") + private String directory; + + @CommandLine.Parameters(arity = "0..*", paramLabel = "FILE", + description = "Files to commit. If not specified, all staged files are committed.") + private List files; + + @CommandLine.ParentCommand + private GitCommand parentCommand; + + @CommandLine.Spec + private CommandLine.Model.CommandSpec spec; + + @Override + public void run() { + List args = new ArrayList<>(); + + if (all) { + args.add("-a"); + } + + if (amend) { + args.add("--amend"); + } + + if (noVerify) { + args.add("--no-verify"); + } + + if (verbose) { + args.add("-v"); + } + + if (quiet) { + args.add("-q"); + } + + if (message != null) { + args.add("-m"); + args.add(message); + } + + if (files != null && !files.isEmpty()) { + args.addAll(files); + } + + File workingDir = parentCommand.getWorkingDirectory(directory); + + GitCommandExecutor.executeGitCommand("commit", args, workingDir, spec); + } +} + diff --git a/gortools/src/main/java/org/gorpipe/gor/cli/git/GitPullCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/git/GitPullCommand.java new file mode 100644 index 00000000..255a9be0 --- /dev/null +++ b/gortools/src/main/java/org/gorpipe/gor/cli/git/GitPullCommand.java @@ -0,0 +1,96 @@ +package org.gorpipe.gor.cli.git; + +import picocli.CommandLine; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; + +@CommandLine.Command(name = "pull", + description = "Fetch from and integrate with another repository or a local branch.") +public class GitPullCommand implements Runnable { + + @CommandLine.Parameters(index = "0", arity = "0..1", paramLabel = "REPOSITORY", + description = "The repository to pull from.") + private String repository; + + @CommandLine.Option(names = {"-d", "--directory"}, paramLabel = "DIRECTORY", + description = "Directory to work in.", defaultValue = ".") + private String directory; + + @CommandLine.Parameters(index = "1", arity = "0..1", paramLabel = "REFSPEC", + description = "The branch or commit to pull.") + private String refspec; + + @CommandLine.Option(names = {"--rebase"}, + description = "Rebase the current branch on top of the upstream branch.") + private boolean rebase; + + @CommandLine.Option(names = {"--no-rebase"}, + description = "Merge the remote-tracking branch into the current branch (default).") + private boolean noRebase; + + @CommandLine.Option(names = {"--ff-only"}, + description = "Refuse to merge unless the current HEAD is already up to date or the merge can be resolved as a fast-forward.") + private boolean ffOnly; + + @CommandLine.Option(names = {"--no-ff"}, + description = "Create a merge commit even when the merge could be resolved as a fast-forward.") + private boolean noff; + + @CommandLine.Option(names = {"-q", "--quiet"}, + description = "Operate quietly.") + private boolean quiet; + + @CommandLine.Option(names = {"-v", "--verbose"}, + description = "Be verbose.") + private boolean verbose; + + @CommandLine.ParentCommand + private GitCommand parentCommand; + + @CommandLine.Spec + private CommandLine.Model.CommandSpec spec; + + @Override + public void run() { + List args = new ArrayList<>(); + + if (rebase) { + args.add("--rebase"); + } + + if (noRebase) { + args.add("--no-rebase"); + } + + if (ffOnly) { + args.add("--ff-only"); + } + + if (noff) { + args.add("--no-ff"); + } + + if (quiet) { + args.add("-q"); + } + + if (verbose) { + args.add("-v"); + } + + if (repository != null) { + args.add(parentCommand.getFullRepositoryPath(repository)); + } + + if (refspec != null) { + args.add(refspec); + } + + File workingDir = parentCommand.getWorkingDirectory(directory); + + GitCommandExecutor.executeGitCommand("pull", args, workingDir, spec); + } +} + diff --git a/gortools/src/main/java/org/gorpipe/gor/cli/git/GitPushCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/git/GitPushCommand.java new file mode 100644 index 00000000..fef0ee75 --- /dev/null +++ b/gortools/src/main/java/org/gorpipe/gor/cli/git/GitPushCommand.java @@ -0,0 +1,96 @@ +package org.gorpipe.gor.cli.git; + +import picocli.CommandLine; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; + +@CommandLine.Command(name = "push", + description = "Update remote refs along with associated objects.") +public class GitPushCommand implements Runnable { + + @CommandLine.Parameters(index = "0", arity = "0..1", paramLabel = "REPOSITORY", + description = "The repository to push to.") + private String repository; + + @CommandLine.Parameters(index = "1", arity = "0..1", paramLabel = "REFSPEC", + description = "The branch or ref to push.") + private String refspec; + + @CommandLine.Option(names = {"-d", "--directory"}, paramLabel = "DIRECTORY", + description = "Directory to work in.", defaultValue = ".") + private String directory; + + @CommandLine.Option(names = {"-u", "--set-upstream"}, + description = "Set upstream for git pull/status.") + private boolean setUpstream; + + @CommandLine.Option(names = {"--force", "-f"}, + description = "Force update remote refs.") + private boolean force; + + @CommandLine.Option(names = {"--force-with-lease"}, + description = "Force update, but refuse to update if the remote-tracking ref has been updated.") + private boolean forceWithLease; + + @CommandLine.Option(names = {"-q", "--quiet"}, + description = "Operate quietly.") + private boolean quiet; + + @CommandLine.Option(names = {"-v", "--verbose"}, + description = "Be verbose.") + private boolean verbose; + + @CommandLine.Option(names = {"--tags"}, + description = "Push all refs under refs/tags.") + private boolean tags; + + @CommandLine.ParentCommand + private GitCommand parentCommand; + + @CommandLine.Spec + private CommandLine.Model.CommandSpec spec; + + @Override + public void run() { + List args = new ArrayList<>(); + + if (setUpstream) { + args.add("-u"); + } + + if (force) { + args.add("--force"); + } + + if (forceWithLease) { + args.add("--force-with-lease"); + } + + if (quiet) { + args.add("-q"); + } + + if (verbose) { + args.add("-v"); + } + + if (tags) { + args.add("--tags"); + } + + if (repository != null) { + args.add(parentCommand.getFullRepositoryPath(repository)); + } + + if (refspec != null) { + args.add(refspec); + } + + File workingDir = parentCommand.getWorkingDirectory(directory); + + GitCommandExecutor.executeGitCommand("push", args, workingDir, spec); + } +} + diff --git a/gortools/src/main/java/org/gorpipe/gor/cli/link/LinkCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/link/LinkCommand.java index 43e1d2aa..a4fb3fc5 100644 --- a/gortools/src/main/java/org/gorpipe/gor/cli/link/LinkCommand.java +++ b/gortools/src/main/java/org/gorpipe/gor/cli/link/LinkCommand.java @@ -12,7 +12,7 @@ public class LinkCommand extends HelpOptions implements Runnable { @CommandLine.ParentCommand - private GorExecCLI gorExecCLI; + private GorExecCLI parentCommand; @Override public void run() { @@ -20,10 +20,10 @@ public void run() { } public String getSecurityContext() { - return gorExecCLI.getSecurityContext(); + return parentCommand.getSecurityContext(); } public String getProjectRoot() { - return gorExecCLI.getProjectRoot(); + return parentCommand.getProjectRoot(); } } diff --git a/gortools/src/main/java/org/gorpipe/gor/cli/link/LinkResolveCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/link/LinkResolveCommand.java index 99bf2f94..1b1329ce 100644 --- a/gortools/src/main/java/org/gorpipe/gor/cli/link/LinkResolveCommand.java +++ b/gortools/src/main/java/org/gorpipe/gor/cli/link/LinkResolveCommand.java @@ -1,6 +1,5 @@ package org.gorpipe.gor.cli.link; -import org.gorpipe.gor.cli.GorExecCLI; import org.gorpipe.gor.driver.linkfile.LinkFile; import org.gorpipe.util.DateUtils; import org.gorpipe.util.Strings; diff --git a/gortools/src/main/java/org/gorpipe/gor/cli/link/LinkRollbackCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/link/LinkRollbackCommand.java index 0495cfa3..04f8c176 100644 --- a/gortools/src/main/java/org/gorpipe/gor/cli/link/LinkRollbackCommand.java +++ b/gortools/src/main/java/org/gorpipe/gor/cli/link/LinkRollbackCommand.java @@ -1,6 +1,5 @@ package org.gorpipe.gor.cli.link; -import org.gorpipe.gor.cli.GorExecCLI; import org.gorpipe.gor.driver.linkfile.LinkFile; import org.gorpipe.util.DateUtils; import picocli.CommandLine; diff --git a/gortools/src/main/java/org/gorpipe/gor/cli/link/LinkUpdateCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/link/LinkUpdateCommand.java index a974b4d3..1875043b 100644 --- a/gortools/src/main/java/org/gorpipe/gor/cli/link/LinkUpdateCommand.java +++ b/gortools/src/main/java/org/gorpipe/gor/cli/link/LinkUpdateCommand.java @@ -1,6 +1,5 @@ package org.gorpipe.gor.cli.link; -import org.gorpipe.gor.cli.GorExecCLI; import org.gorpipe.gor.driver.linkfile.LinkFile; import org.gorpipe.util.Strings; import picocli.CommandLine; diff --git a/gortools/src/main/scala/gorsat/InputSources/Exec.scala b/gortools/src/main/scala/gorsat/InputSources/Exec.scala index c330864c..ce34854f 100644 --- a/gortools/src/main/scala/gorsat/InputSources/Exec.scala +++ b/gortools/src/main/scala/gorsat/InputSources/Exec.scala @@ -22,17 +22,15 @@ package gorsat.InputSources -import gorsat.Commands.{CommandArguments, CommandParseUtilities, InputSourceInfo, InputSourceParsingResult} -import gorsat.Iterators.{CountingNorRowIterator, RowListIterator} +import gorsat.Commands.{CommandArguments, InputSourceInfo, InputSourceParsingResult} +import gorsat.Iterators.{RowListIterator} import org.gorpipe.gor.cli.GorExecCLI -import org.gorpipe.gor.cli.link.LinkUpdateCommand -import org.gorpipe.gor.model.{GorCommand, NoValidateRowBase, Row} +import org.gorpipe.gor.model.{NoValidateRowBase, Row} import org.gorpipe.gor.session.GorContext import picocli.CommandLine -import java.io.{ByteArrayOutputStream, PrintStream, PrintWriter} +import java.io.{ByteArrayOutputStream, PrintStream} import scala.collection.mutable.ListBuffer -import scala.util.Using /** * Execute selected gor commands in NOR context. From 582ebf66c72e328257a1437ae89ea47638209b79 Mon Sep 17 00:00:00 2001 From: Gisli Magnusson Date: Mon, 1 Dec 2025 22:42:43 -0500 Subject: [PATCH 07/17] feat(ENGKNOW-2892): Add gor query support for link files in gor server. --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6b2aa47d..5c19bcfb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -142,7 +142,7 @@ jobs: files: '**/TEST-*.xml' publishSnapshot: - if: ${{ github.ref == 'refs/heads/main' }} + #if: ${{ github.ref == 'refs/heads/main' }} needs: [test, slowTest, integrationTest] runs-on: ubuntu-latest steps: From 9b12410083f894e295eb5d36ee362f3c31c04b06 Mon Sep 17 00:00:00 2001 From: Gisli Magnusson Date: Mon, 1 Dec 2025 23:11:20 -0500 Subject: [PATCH 08/17] feat(ENGKNOW-2892): Add gor query support for link files in gor server. --- .../gorpipe/gor/cli/link/LinkCommandTest.java | 50 ++++++++++--------- 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/gorscripts/src/test/java/org/gorpipe/gor/cli/link/LinkCommandTest.java b/gorscripts/src/test/java/org/gorpipe/gor/cli/link/LinkCommandTest.java index d8a519fb..5bee6e6a 100644 --- a/gorscripts/src/test/java/org/gorpipe/gor/cli/link/LinkCommandTest.java +++ b/gorscripts/src/test/java/org/gorpipe/gor/cli/link/LinkCommandTest.java @@ -5,9 +5,11 @@ import java.nio.file.Path; import java.time.Instant; +import org.gorpipe.gor.cli.GorCLI; import org.gorpipe.gor.driver.linkfile.LinkFile; import org.gorpipe.gor.driver.providers.stream.sources.file.FileSource; import static org.junit.Assert.assertEquals; + import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -22,9 +24,9 @@ public class LinkCommandTest { @Test public void testUpdateCreatesLinkFileAndAppliesHeaders() throws Exception { Path linkFile = temp.getRoot().toPath().resolve("update_test.gor.link"); - CommandLine cmd = new CommandLine(new LinkCommand()); + CommandLine cmd = new CommandLine(new GorCLI()); - int exitCode = cmd.execute("update", linkFile.toString(), "data/file1.gor", "-h", "ENTRIES_COUNT_MAX=5"); + int exitCode = cmd.execute("link", "update", linkFile.toString(), "data/file1.gor", "-h", "ENTRIES_COUNT_MAX=5"); assertEquals(0, exitCode); LinkFile link = LinkFile.load(new FileSource(linkFile)); @@ -36,9 +38,9 @@ public void testUpdateCreatesLinkFileAndAppliesHeaders() throws Exception { @Test public void testUpdateWithMd5AndInfo() throws Exception { Path linkFile = temp.getRoot().toPath().resolve("update_md5_info.gor.link"); - CommandLine cmd = new CommandLine(new LinkCommand()); + CommandLine cmd = new CommandLine(new GorCLI()); - int exitCode = cmd.execute("update", linkFile.toString(), "data/file1.gor", + int exitCode = cmd.execute("link", "update", linkFile.toString(), "data/file1.gor", "-m", "abc123", "-i", "first entry"); assertEquals(0, exitCode); @@ -51,13 +53,13 @@ public void testUpdateWithMd5AndInfo() throws Exception { @Test public void testRollbackLatestEntry() throws Exception { Path linkFile = temp.getRoot().toPath().resolve("rollback_latest.gor.link"); - CommandLine cmd = new CommandLine(new LinkCommand()); + CommandLine cmd = new CommandLine(new GorCLI()); - cmd.execute("update", linkFile.toString(), "data/file1.gor"); + cmd.execute("link", "update", linkFile.toString(), "data/file1.gor"); Thread.sleep(5); - cmd.execute("update", linkFile.toString(), "data/file2.gor"); + cmd.execute("link", "update", linkFile.toString(), "data/file2.gor"); - int exitCode = cmd.execute("rollback", linkFile.toString()); + int exitCode = cmd.execute("link", "rollback", linkFile.toString()); assertEquals(0, exitCode); LinkFile link = LinkFile.load(new FileSource(linkFile)); @@ -68,17 +70,17 @@ public void testRollbackLatestEntry() throws Exception { @Test public void testRollbackToDate() throws Exception { Path linkFile = temp.getRoot().toPath().resolve("rollback_date.gor.link"); - CommandLine cmd = new CommandLine(new LinkCommand()); + CommandLine cmd = new CommandLine(new GorCLI()); - cmd.execute("update", linkFile.toString(), "data/file1.gor"); + cmd.execute("link", "update", linkFile.toString(), "data/file1.gor"); LinkFile first = LinkFile.load(new FileSource(linkFile)); long firstTimestamp = first.getLatestEntry().timestamp(); Thread.sleep(5); - cmd.execute("update", linkFile.toString(), "data/file2.gor"); + cmd.execute("link", "update", linkFile.toString(), "data/file2.gor"); String rollbackIso = Instant.ofEpochMilli(firstTimestamp).toString(); - int exitCode = cmd.execute("rollback", linkFile.toString(), "-d", rollbackIso); + int exitCode = cmd.execute("link", "rollback", linkFile.toString(), "-d", rollbackIso); assertEquals(0, exitCode); LinkFile link = LinkFile.load(new FileSource(linkFile)); @@ -89,27 +91,27 @@ public void testRollbackToDate() throws Exception { @Test public void testResolveLatestEntry() throws Exception { Path linkFile = temp.getRoot().toPath().resolve("resolve_latest.gor.link"); - CommandLine cmd = new CommandLine(new LinkCommand()); + CommandLine cmd = new CommandLine(new GorCLI()); - cmd.execute("update", linkFile.toString(), "data/file1.gor"); + cmd.execute("link", "update", linkFile.toString(), "data/file1.gor"); Thread.sleep(5); - cmd.execute("update", linkFile.toString(), "data/file2.gor"); + cmd.execute("link", "update", linkFile.toString(), "data/file2.gor"); - String resolved = executeAndCapture(cmd, "resolve", linkFile.toString()); + String resolved = executeAndCapture(cmd, "link", "resolve", linkFile.toString()); assertEquals(resolve(linkFile, "data/file2.gor"), resolved); } @Test public void testResolveSpecificDate() throws Exception { Path linkFile = temp.getRoot().toPath().resolve("resolve_date.gor.link"); - CommandLine cmd = new CommandLine(new LinkCommand()); + CommandLine cmd = new CommandLine(new GorCLI()); - cmd.execute("update", linkFile.toString(), "data/file1.gor"); + cmd.execute("link", "update", linkFile.toString(), "data/file1.gor"); long firstTimestamp = LinkFile.load(new FileSource(linkFile)).getLatestEntry().timestamp(); Thread.sleep(5); - cmd.execute("update", linkFile.toString(), "data/file2.gor"); + cmd.execute("link", "update", linkFile.toString(), "data/file2.gor"); - String resolved = executeAndCapture(cmd, "resolve", linkFile.toString(), + String resolved = executeAndCapture(cmd, "link", "resolve", linkFile.toString(), "-d", Instant.ofEpochMilli(firstTimestamp).toString()); assertEquals(resolve(linkFile, "data/file1.gor"), resolved); } @@ -117,14 +119,14 @@ public void testResolveSpecificDate() throws Exception { @Test public void testResolveFullEntry() throws Exception { Path linkFile = temp.getRoot().toPath().resolve("resolve_full.gor.link"); - CommandLine cmd = new CommandLine(new LinkCommand()); + CommandLine cmd = new CommandLine(new GorCLI()); - cmd.execute("update", linkFile.toString(), "data/file1.gor"); + cmd.execute("link", "update", linkFile.toString(), "data/file1.gor"); Thread.sleep(5); - cmd.execute("update", linkFile.toString(), "data/file2.gor"); + cmd.execute("link", "update", linkFile.toString(), "data/file2.gor"); String expectedEntry = LinkFile.load(new FileSource(linkFile)).getLatestEntry().format(); - String resolved = executeAndCapture(cmd, "resolve", linkFile.toString(), "-f"); + String resolved = executeAndCapture(cmd, "link", "resolve", linkFile.toString(), "-f"); assertEquals(expectedEntry, resolved); } From 30e7c9af305516d62fe06e91e419707817aa78a8 Mon Sep 17 00:00:00 2001 From: Gisli Magnusson Date: Tue, 2 Dec 2025 00:16:46 -0500 Subject: [PATCH 09/17] feat(ENGKNOW-2892): Add gor query support for link files in gor server. --- .../gor/cli/link/LinkResolveCommand.java | 2 +- .../main/scala/gorsat/InputSources/Exec.scala | 18 ++++++++++++++++-- .../src/test/java/gorsat/Inputs/UTestLink.java | 10 +++++----- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/gortools/src/main/java/org/gorpipe/gor/cli/link/LinkResolveCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/link/LinkResolveCommand.java index 1b1329ce..223d3300 100644 --- a/gortools/src/main/java/org/gorpipe/gor/cli/link/LinkResolveCommand.java +++ b/gortools/src/main/java/org/gorpipe/gor/cli/link/LinkResolveCommand.java @@ -40,7 +40,7 @@ public void run() { } String output; if (returnFullEntry) { - output = entry.format(); + output = entry.format().replace('\t', ' '); } else { var resolved = linkFile.getEntryUrl(timestamp); if (Strings.isNullOrEmpty(resolved)) { diff --git a/gortools/src/main/scala/gorsat/InputSources/Exec.scala b/gortools/src/main/scala/gorsat/InputSources/Exec.scala index ce34854f..0ba73121 100644 --- a/gortools/src/main/scala/gorsat/InputSources/Exec.scala +++ b/gortools/src/main/scala/gorsat/InputSources/Exec.scala @@ -42,7 +42,7 @@ class Exec() extends InputSourceInfo("EXEC", CommandArguments("","", 2, 10, igno val result = processExec(context, argString, iargs.slice(1, iargs.length), args) - val header = "Chrom\tpos\t" + result(0) + val header = "ChromNor\tPosNor\t" + result(0) val myHeaderLength = header.split("\t").length val lineList = new ListBuffer[Row]() for (row <- result.slice(1, result.length)) { @@ -92,7 +92,21 @@ class Exec() extends InputSourceInfo("EXEC", CommandArguments("","", 2, 10, igno std_ps.close() err_ps.close() } + var retArray = Array("Status\tStdOut\tStdErr") + val stdLines = std_baos.toString.stripLineEnd.split("\n") + val errLines = err_baos.toString.stripLineEnd.split("\n") + if (stdLines.length > errLines.length) { + for (i <- stdLines.indices) { + val errLine = if (i < errLines.length) errLines(i) else "" + retArray :+= exitCode + "\t" + stdLines(i) + "\t\"" + errLine + "\"" + } + } else { + for (i <- errLines.indices) { + val stdLine = if (i < stdLines.length) stdLines(i) else "" + retArray :+= exitCode + "\t" + stdLine + "\t" + errLines(i) + "\"" + } + } - Array("Status\tStdOut\tStdErr", exitCode + "\t\"" + std_baos.toString.stripLineEnd + "\"\t\"" + err_baos.toString.stripLineEnd + "\"") + retArray } } diff --git a/gortools/src/test/java/gorsat/Inputs/UTestLink.java b/gortools/src/test/java/gorsat/Inputs/UTestLink.java index 3f0b430d..8ad01b9e 100644 --- a/gortools/src/test/java/gorsat/Inputs/UTestLink.java +++ b/gortools/src/test/java/gorsat/Inputs/UTestLink.java @@ -23,7 +23,7 @@ public void testLinkHelp() throws Exception { String res = TestUtils.runGorPipe("exec gor link --help "); assertEquals("0", res.split("\n")[1].split("\t")[2]); - assertTrue(res.split("\n")[1].split("\t")[3].startsWith("\"Link file management commands.")); + assertTrue(res.split("\n")[1].split("\t")[3].startsWith("Link file management commands.")); } @Test @@ -35,7 +35,7 @@ public void testUpdateCreatesLinkFileError() throws Exception { LinkFile link = LinkFile.load(new FileSource(linkFile)); assertEquals(0, link.getEntriesCount()); assertEquals("2", res.split("\n")[1].split("\t")[2]); - assertEquals("\"Unknown options: '-xxx', 'ENTRIES_COUNT_MAX=5'", res.split("\n")[1].split("\t")[3]); + assertEquals("Unknown options: '-xxx', 'ENTRIES_COUNT_MAX=5'", res.split("\n")[1].split("\t")[3]); } @Test @@ -123,7 +123,7 @@ public void testResolveLatestEntry() throws Exception { String res = TestUtils.runGorPipe("exec gor link resolve " + linkFile); assertEquals("0", res.split("\n")[1].split("\t")[2]); - assertEquals("\"" + resolve(linkFile, "data/file2.gor") + "\"", res.split("\n")[1].split("\t")[3]); + assertEquals(resolve(linkFile, "data/file2.gor"), res.split("\n")[1].split("\t")[3]); } @Test @@ -139,7 +139,7 @@ public void testResolveSpecificDate() throws Exception { String res = TestUtils.runGorPipe("exec gor link resolve " + linkFile.toString() + " -d " + Instant.ofEpochMilli(firstTimestamp).toString()); assertEquals("0", res.split("\n")[1].split("\t")[2]); - assertEquals("\"" + resolve(linkFile, "data/file1.gor") + "\"", res.split("\n")[1].split("\t")[3]); + assertEquals(resolve(linkFile, "data/file1.gor"), res.split("\n")[1].split("\t")[3]); } @Test @@ -155,7 +155,7 @@ public void testResolveFullEntry() throws Exception { String res = TestUtils.runGorPipe("exec gor link resolve " + linkFile.toString() + " -f"); assertEquals("0", res.split("\n")[1].split("\t")[2]); - assertEquals("\"" + expectedEntry + "\"", CommandParseUtilities.quoteSafeSplit(res.split("\n")[1], '\t')[3]); + assertEquals(expectedEntry.replace('\t', ' '), CommandParseUtilities.quoteSafeSplit(res.split("\n")[1], '\t')[3]); } private String resolve(Path linkFile, String relative) { From 26925fb9b0ebd85b6a1342e24d58d32e7ffc5246 Mon Sep 17 00:00:00 2001 From: Gisli Magnusson Date: Tue, 2 Dec 2025 00:27:31 -0500 Subject: [PATCH 10/17] feat(ENGKNOW-2892): Add gor query support for link files in gor server. --- .../src/main/java/org/gorpipe/gor/cli/git/GitCloneCommand.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gortools/src/main/java/org/gorpipe/gor/cli/git/GitCloneCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/git/GitCloneCommand.java index 821d0794..df3bf4ff 100644 --- a/gortools/src/main/java/org/gorpipe/gor/cli/git/GitCloneCommand.java +++ b/gortools/src/main/java/org/gorpipe/gor/cli/git/GitCloneCommand.java @@ -76,7 +76,7 @@ public void run() { args.add(directory); } - File workingDir = parentCommand.getWorkingDirectory(directory); + File workingDir = parentCommand.getWorkingDirectory("."); GitCommandExecutor.executeGitCommand("clone", args, workingDir, spec); } From 58d305022e996ddb8f44c8c051a5f0c5a4bb6bd8 Mon Sep 17 00:00:00 2001 From: Gisli Magnusson Date: Tue, 2 Dec 2025 00:38:28 -0500 Subject: [PATCH 11/17] feat(ENGKNOW-2892): Add gor query support for link files in gor server. --- .../src/test/java/org/gorpipe/gor/cli/link/LinkCommandTest.java | 2 +- gortools/src/main/java/org/gorpipe/gor/cli/git/GitCommand.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/gorscripts/src/test/java/org/gorpipe/gor/cli/link/LinkCommandTest.java b/gorscripts/src/test/java/org/gorpipe/gor/cli/link/LinkCommandTest.java index 5bee6e6a..226ea9af 100644 --- a/gorscripts/src/test/java/org/gorpipe/gor/cli/link/LinkCommandTest.java +++ b/gorscripts/src/test/java/org/gorpipe/gor/cli/link/LinkCommandTest.java @@ -127,7 +127,7 @@ public void testResolveFullEntry() throws Exception { String expectedEntry = LinkFile.load(new FileSource(linkFile)).getLatestEntry().format(); String resolved = executeAndCapture(cmd, "link", "resolve", linkFile.toString(), "-f"); - assertEquals(expectedEntry, resolved); + assertEquals(expectedEntry.replace('\t', ' '), resolved); } private String resolve(Path linkFile, String relative) { diff --git a/gortools/src/main/java/org/gorpipe/gor/cli/git/GitCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/git/GitCommand.java index 348c1447..b305ca2a 100644 --- a/gortools/src/main/java/org/gorpipe/gor/cli/git/GitCommand.java +++ b/gortools/src/main/java/org/gorpipe/gor/cli/git/GitCommand.java @@ -36,7 +36,7 @@ public String getFullRepositoryPath(String repository) { return repository; } String user = System.getenv("GOR_GIT_USER"); - String pass = System.getenv("GOR_GIT_PASS"); + String pass = System.getenv("GOR_GIT_TOKEN"); if (user != null && pass != null) { var userPass = "%s:%s@".formatted(user, pass); return "https://%sgithub.com/GeneDx/%s.git".formatted(userPass, repository); From bb72e3d67e38f4e44f9434a1808a1980497d5e09 Mon Sep 17 00:00:00 2001 From: Gisli Magnusson Date: Tue, 2 Dec 2025 01:22:46 -0500 Subject: [PATCH 12/17] feat(ENGKNOW-2892): Add gor query support for link files in gor server. --- .../gorpipe/gor/cli/git/GitAddCommand.java | 88 +++++++++++++++ .../org/gorpipe/gor/cli/git/GitCommand.java | 4 +- .../gorpipe/gor/cli/git/GitConfigCommand.java | 104 ++++++++++++++++++ 3 files changed, 194 insertions(+), 2 deletions(-) create mode 100644 gortools/src/main/java/org/gorpipe/gor/cli/git/GitAddCommand.java create mode 100644 gortools/src/main/java/org/gorpipe/gor/cli/git/GitConfigCommand.java diff --git a/gortools/src/main/java/org/gorpipe/gor/cli/git/GitAddCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/git/GitAddCommand.java new file mode 100644 index 00000000..58fddd85 --- /dev/null +++ b/gortools/src/main/java/org/gorpipe/gor/cli/git/GitAddCommand.java @@ -0,0 +1,88 @@ +package org.gorpipe.gor.cli.git; + +import picocli.CommandLine; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; + +@CommandLine.Command(name = "add", + description = "Add file contents to the staging area.") +public class GitAddCommand implements Runnable { + + @CommandLine.Parameters(arity = "0..*", paramLabel = "FILE", + description = "Files to add to the staging area. Patterns can be used, e.g., '*.java'.") + private List files; + + @CommandLine.Option(names = {"-A", "--all"}, + description = "Stage all changes in the working tree.") + private boolean all; + + @CommandLine.Option(names = {"-u", "--update"}, + description = "Stage only tracked files that have been modified or deleted.") + private boolean update; + + @CommandLine.Option(names = {"-f", "--force"}, + description = "Allow adding otherwise ignored files.") + private boolean force; + + @CommandLine.Option(names = {"-v", "--verbose"}, + description = "Be verbose.") + private boolean verbose; + + @CommandLine.Option(names = {"-n", "--dry-run"}, + description = "Don't actually add the file(s), just show if they exist and/or will be ignored.") + private boolean dryRun; + + @CommandLine.Option(names = {"-p", "--patch"}, + description = "Interactively choose hunks of patch between the index and the work tree.") + private boolean patch; + + @CommandLine.Option(names = {"-d", "--directory"}, paramLabel = "DIRECTORY", + description = "Directory to work in.", defaultValue = ".") + private String directory; + + @CommandLine.ParentCommand + private GitCommand parentCommand; + + @CommandLine.Spec + private CommandLine.Model.CommandSpec spec; + + @Override + public void run() { + List args = new ArrayList<>(); + + if (all) { + args.add("-A"); + } + + if (update) { + args.add("-u"); + } + + if (force) { + args.add("-f"); + } + + if (verbose) { + args.add("-v"); + } + + if (dryRun) { + args.add("-n"); + } + + if (patch) { + args.add("-p"); + } + + if (files != null && !files.isEmpty()) { + args.addAll(files); + } + + File workingDir = parentCommand.getWorkingDirectory(directory); + + GitCommandExecutor.executeGitCommand("add", args, workingDir, spec); + } +} + diff --git a/gortools/src/main/java/org/gorpipe/gor/cli/git/GitCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/git/GitCommand.java index b305ca2a..008f1e5e 100644 --- a/gortools/src/main/java/org/gorpipe/gor/cli/git/GitCommand.java +++ b/gortools/src/main/java/org/gorpipe/gor/cli/git/GitCommand.java @@ -10,9 +10,9 @@ @SuppressWarnings("squid:S106") @CommandLine.Command(name = "git", - description = "Wrapper for git commands (clone, checkout, pull, push, commit).", + description = "Wrapper for git commands (clone, checkout, pull, push, commit, add, config).", header = "Git command wrapper.", - subcommands = {GitCloneCommand.class, GitCheckoutCommand.class, GitPullCommand.class, GitPushCommand.class, GitCommitCommand.class}) + subcommands = {GitCloneCommand.class, GitCheckoutCommand.class, GitPullCommand.class, GitPushCommand.class, GitCommitCommand.class, GitAddCommand.class, GitConfigCommand.class}) public class GitCommand extends HelpOptions implements Runnable { @CommandLine.ParentCommand diff --git a/gortools/src/main/java/org/gorpipe/gor/cli/git/GitConfigCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/git/GitConfigCommand.java new file mode 100644 index 00000000..bbf91909 --- /dev/null +++ b/gortools/src/main/java/org/gorpipe/gor/cli/git/GitConfigCommand.java @@ -0,0 +1,104 @@ +package org.gorpipe.gor.cli.git; + +import picocli.CommandLine; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; + +@CommandLine.Command(name = "config", + description = "Get and set repository or global options.") +public class GitConfigCommand implements Runnable { + + @CommandLine.Parameters(arity = "0..*", paramLabel = "KEY [VALUE]", + description = "Configuration key, optionally followed by value to set.") + private List configArgs; + + @CommandLine.Option(names = {"--global"}, + description = "Use global config file.") + private boolean global; + + @CommandLine.Option(names = {"--local"}, + description = "Use repository config file (default).") + private boolean local; + + @CommandLine.Option(names = {"--system"}, + description = "Use system config file.") + private boolean system; + + @CommandLine.Option(names = {"--list", "-l"}, + description = "List all variables set in config file.") + private boolean list; + + @CommandLine.Option(names = {"--get"}, + description = "Get the value for a given key.") + private boolean get; + + @CommandLine.Option(names = {"--set"}, + description = "Set a value for a given key.") + private boolean set; + + @CommandLine.Option(names = {"--unset"}, + description = "Remove a key from the config file.") + private boolean unset; + + @CommandLine.Option(names = {"--unset-all"}, + description = "Remove all matches for a key from the config file.") + private boolean unsetAll; + + @CommandLine.Option(names = {"-d", "--directory"}, paramLabel = "DIRECTORY", + description = "Directory to work in.", defaultValue = ".") + private String directory; + + @CommandLine.ParentCommand + private GitCommand parentCommand; + + @CommandLine.Spec + private CommandLine.Model.CommandSpec spec; + + @Override + public void run() { + List args = new ArrayList<>(); + + if (global) { + args.add("--global"); + } + + if (local) { + args.add("--local"); + } + + if (system) { + args.add("--system"); + } + + if (list) { + args.add("--list"); + } + + if (get) { + args.add("--get"); + } + + if (set) { + args.add("--set"); + } + + if (unset) { + args.add("--unset"); + } + + if (unsetAll) { + args.add("--unset-all"); + } + + if (configArgs != null && !configArgs.isEmpty()) { + args.addAll(configArgs); + } + + File workingDir = parentCommand.getWorkingDirectory(directory); + + GitCommandExecutor.executeGitCommand("config", args, workingDir, spec); + } +} + From 5eb12e97e228abcea90c256e7f7b55669cb255a9 Mon Sep 17 00:00:00 2001 From: Gisli Magnusson Date: Tue, 2 Dec 2025 13:25:07 -0500 Subject: [PATCH 13/17] feat(ENGKNOW-2892): Add gor query support for link files in gor server. --- gortools/src/main/scala/gorsat/InputSources/Exec.scala | 4 ++-- gortools/src/main/scala/gorsat/process/GorPrePipe.scala | 7 ++++++- model/src/main/scala/gorsat/gorsatDynIterator.scala | 2 ++ 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/gortools/src/main/scala/gorsat/InputSources/Exec.scala b/gortools/src/main/scala/gorsat/InputSources/Exec.scala index 0ba73121..8c2db7f0 100644 --- a/gortools/src/main/scala/gorsat/InputSources/Exec.scala +++ b/gortools/src/main/scala/gorsat/InputSources/Exec.scala @@ -98,12 +98,12 @@ class Exec() extends InputSourceInfo("EXEC", CommandArguments("","", 2, 10, igno if (stdLines.length > errLines.length) { for (i <- stdLines.indices) { val errLine = if (i < errLines.length) errLines(i) else "" - retArray :+= exitCode + "\t" + stdLines(i) + "\t\"" + errLine + "\"" + retArray :+= exitCode + "\t" + stdLines(i) + "\t" + errLine } } else { for (i <- errLines.indices) { val stdLine = if (i < stdLines.length) stdLines(i) else "" - retArray :+= exitCode + "\t" + stdLine + "\t" + errLines(i) + "\"" + retArray :+= exitCode + "\t" + stdLine + "\t" + errLines(i) } } diff --git a/gortools/src/main/scala/gorsat/process/GorPrePipe.scala b/gortools/src/main/scala/gorsat/process/GorPrePipe.scala index 560c27f8..7619bf7e 100644 --- a/gortools/src/main/scala/gorsat/process/GorPrePipe.scala +++ b/gortools/src/main/scala/gorsat/process/GorPrePipe.scala @@ -84,7 +84,12 @@ object GorPrePipe { || pipeSteps(0).toUpperCase.trim.startsWith("GORIF") || pipeSteps(0).toUpperCase.trim.startsWith("NORIF") || pipeSteps(0).toUpperCase.trim.startsWith("SPARK") - || pipeSteps(0).toUpperCase.trim.startsWith("SELECT"))) + || pipeSteps(0).toUpperCase.trim.startsWith("SELECT") + || pipeSteps(0).toUpperCase.trim.startsWith("META") + || pipeSteps(0).toUpperCase.trim.startsWith("EXEC") + || pipeSteps(0).toUpperCase.trim.startsWith("NORROWS") + || pipeSteps(0).toUpperCase.trim.startsWith("GORROWS") + )) "GOR " + pipeSteps(i).trim else pipeSteps(i).trim diff --git a/model/src/main/scala/gorsat/gorsatDynIterator.scala b/model/src/main/scala/gorsat/gorsatDynIterator.scala index c3d49fd8..3d6716cd 100644 --- a/model/src/main/scala/gorsat/gorsatDynIterator.scala +++ b/model/src/main/scala/gorsat/gorsatDynIterator.scala @@ -58,6 +58,8 @@ def addStartSelector(cmd : String, seekChr : String, seekPos : Int, endPos : Int else if (cmd.toLowerCase.startsWith("norrows ")) return cmd else if (cmd.toLowerCase.startsWith("sdl ")) return cmd else if (cmd.toLowerCase.startsWith("norsql ")) return cmd + else if (cmd.toLowerCase.startsWith("meta ")) return cmd + else if (cmd.toLowerCase.startsWith("exec ")) return cmd var cmd2 = cmd val lcmd = cmd2.toLowerCase From dc49e1967867781ff7c2027f64282adb58c7243d Mon Sep 17 00:00:00 2001 From: Gisli Magnusson Date: Tue, 2 Dec 2025 15:37:12 -0500 Subject: [PATCH 14/17] feat(ENGKNOW-2892): Add gor query support for link files in gor server. --- .../main/scala/gorsat/InputSources/Exec.scala | 25 ++++++++--------- .../test/java/gorsat/Inputs/UTestLink.java | 28 ++++++++----------- 2 files changed, 22 insertions(+), 31 deletions(-) diff --git a/gortools/src/main/scala/gorsat/InputSources/Exec.scala b/gortools/src/main/scala/gorsat/InputSources/Exec.scala index 8c2db7f0..8db82e16 100644 --- a/gortools/src/main/scala/gorsat/InputSources/Exec.scala +++ b/gortools/src/main/scala/gorsat/InputSources/Exec.scala @@ -23,7 +23,8 @@ package gorsat.InputSources import gorsat.Commands.{CommandArguments, InputSourceInfo, InputSourceParsingResult} -import gorsat.Iterators.{RowListIterator} +import gorsat.Iterators.RowListIterator +import org.gorpipe.exceptions.GorParsingException import org.gorpipe.gor.cli.GorExecCLI import org.gorpipe.gor.model.{NoValidateRowBase, Row} import org.gorpipe.gor.session.GorContext @@ -92,21 +93,17 @@ class Exec() extends InputSourceInfo("EXEC", CommandArguments("","", 2, 10, igno std_ps.close() err_ps.close() } - var retArray = Array("Status\tStdOut\tStdErr") - val stdLines = std_baos.toString.stripLineEnd.split("\n") - val errLines = err_baos.toString.stripLineEnd.split("\n") - if (stdLines.length > errLines.length) { - for (i <- stdLines.indices) { - val errLine = if (i < errLines.length) errLines(i) else "" - retArray :+= exitCode + "\t" + stdLines(i) + "\t" + errLine - } + if (exitCode != 0) { + throw new GorParsingException(s"EXEC command: ${argString} failed with exit code: ${exitCode} and output:\n${std_baos.toString}\n${err_baos.toString}") + } + var stdLines = std_baos.toString.stripLineEnd.split("\n") + + if (stdLines.size == 0 || !stdLines(0).startsWith("#")) { + stdLines = stdLines.prepended("Result") } else { - for (i <- errLines.indices) { - val stdLine = if (i < stdLines.length) stdLines(i) else "" - retArray :+= exitCode + "\t" + stdLine + "\t" + errLines(i) - } + stdLines(0) = stdLines(0).substring(1) } - retArray + stdLines } } diff --git a/gortools/src/test/java/gorsat/Inputs/UTestLink.java b/gortools/src/test/java/gorsat/Inputs/UTestLink.java index 8ad01b9e..9026be77 100644 --- a/gortools/src/test/java/gorsat/Inputs/UTestLink.java +++ b/gortools/src/test/java/gorsat/Inputs/UTestLink.java @@ -2,6 +2,7 @@ import gorsat.Commands.CommandParseUtilities; import gorsat.TestUtils; +import org.gorpipe.exceptions.GorParsingException; import org.gorpipe.gor.driver.linkfile.LinkFile; import org.gorpipe.gor.driver.providers.stream.sources.file.FileSource; import org.junit.Rule; @@ -22,20 +23,21 @@ public class UTestLink { public void testLinkHelp() throws Exception { String res = TestUtils.runGorPipe("exec gor link --help "); - assertEquals("0", res.split("\n")[1].split("\t")[2]); - assertTrue(res.split("\n")[1].split("\t")[3].startsWith("Link file management commands.")); + assertTrue(res.split("\n")[1].split("\t")[2].startsWith("Link file management commands.")); } @Test public void testUpdateCreatesLinkFileError() throws Exception { Path linkFile = temp.getRoot().toPath().resolve("update_test.gor.link"); - String res = TestUtils.runGorPipe("exec gor link update " + linkFile.toString() + " data/file1.gor -xxx ENTRIES_COUNT_MAX=5"); - + try { + String res = TestUtils.runGorPipe("exec gor link update " + linkFile.toString() + " data/file1.gor -xxx ENTRIES_COUNT_MAX=5"); + } catch (GorParsingException e) { + String res = e.getMessage(); + assertTrue(res.contains("Unknown options: '-xxx', 'ENTRIES_COUNT_MAX=5'")); + } LinkFile link = LinkFile.load(new FileSource(linkFile)); assertEquals(0, link.getEntriesCount()); - assertEquals("2", res.split("\n")[1].split("\t")[2]); - assertEquals("Unknown options: '-xxx', 'ENTRIES_COUNT_MAX=5'", res.split("\n")[1].split("\t")[3]); } @Test @@ -45,7 +47,6 @@ public void testUpdateCreatesLinkFileAndAppliesHeaders() throws Exception { String res = TestUtils.runGorPipe("exec gor link update " + linkFile.toString() + " data/file1.gor -h ENTRIES_COUNT_MAX=5"); LinkFile link = LinkFile.load(new FileSource(linkFile)); - assertEquals("0", res.split("\n")[1].split("\t")[2]); assertEquals(1, link.getEntriesCount()); assertEquals(resolve(linkFile, "data/file1.gor"), link.getLatestEntryUrl()); assertEquals(5, link.getEntriesCountMax()); @@ -58,7 +59,6 @@ public void testUpdateCreatesLinkFileWithProjectRooot() throws Exception { String res = TestUtils.runGorPipe("exec gor link update update_test.gor.link data/file1.gor -h ENTRIES_COUNT_MAX=5", temp.getRoot().toPath().toString() , true, "dummy"); LinkFile link = LinkFile.load(new FileSource(linkFile)); - assertEquals("0", res.split("\n")[1].split("\t")[2]); assertEquals(1, link.getEntriesCount()); assertEquals(resolve(linkFile, "data/file1.gor"), link.getLatestEntryUrl()); assertEquals(5, link.getEntriesCountMax()); @@ -72,7 +72,6 @@ public void testUpdateWithMd5AndInfo() throws Exception { LinkFile link = LinkFile.load(new FileSource(linkFile)); var latest = link.getLatestEntry(); - assertEquals("0", res.split("\n")[1].split("\t")[2]); assertEquals("abc123", latest.md5()); assertEquals("'first entry'", latest.info()); } @@ -87,7 +86,6 @@ public void testRollbackLatestEntry() throws Exception { String res = TestUtils.runGorPipe("exec gor link rollback " + linkFile.toString()); - assertEquals("0", res.split("\n")[1].split("\t")[2]); LinkFile link = LinkFile.load(new FileSource(linkFile)); assertEquals(1, link.getEntriesCount()); assertEquals(resolve(linkFile, "data/file1.gor"), link.getLatestEntryUrl()); @@ -106,7 +104,6 @@ public void testRollbackToDate() throws Exception { String rollbackIso = Instant.ofEpochMilli(firstTimestamp).toString(); String res = TestUtils.runGorPipe("exec gor link rollback " + linkFile.toString() + " -d " + rollbackIso); - assertEquals("0", res.split("\n")[1].split("\t")[2]); LinkFile link = LinkFile.load(new FileSource(linkFile)); assertEquals(1, link.getEntriesCount()); assertEquals(resolve(linkFile, "data/file1.gor"), link.getLatestEntryUrl()); @@ -122,8 +119,7 @@ public void testResolveLatestEntry() throws Exception { String res = TestUtils.runGorPipe("exec gor link resolve " + linkFile); - assertEquals("0", res.split("\n")[1].split("\t")[2]); - assertEquals(resolve(linkFile, "data/file2.gor"), res.split("\n")[1].split("\t")[3]); + assertEquals(resolve(linkFile, "data/file2.gor"), res.split("\n")[1].split("\t")[2]); } @Test @@ -138,8 +134,7 @@ public void testResolveSpecificDate() throws Exception { String res = TestUtils.runGorPipe("exec gor link resolve " + linkFile.toString() + " -d " + Instant.ofEpochMilli(firstTimestamp).toString()); - assertEquals("0", res.split("\n")[1].split("\t")[2]); - assertEquals(resolve(linkFile, "data/file1.gor"), res.split("\n")[1].split("\t")[3]); + assertEquals(resolve(linkFile, "data/file1.gor"), res.split("\n")[1].split("\t")[2]); } @Test @@ -154,8 +149,7 @@ public void testResolveFullEntry() throws Exception { String res = TestUtils.runGorPipe("exec gor link resolve " + linkFile.toString() + " -f"); - assertEquals("0", res.split("\n")[1].split("\t")[2]); - assertEquals(expectedEntry.replace('\t', ' '), CommandParseUtilities.quoteSafeSplit(res.split("\n")[1], '\t')[3]); + assertEquals(expectedEntry.replace('\t', ' '), CommandParseUtilities.quoteSafeSplit(res.split("\n")[1], '\t')[2]); } private String resolve(Path linkFile, String relative) { From 6155eff1e7ee80c033e30c4d305aec9e3bda8895 Mon Sep 17 00:00:00 2001 From: Gisli Magnusson Date: Wed, 3 Dec 2025 16:01:30 +0000 Subject: [PATCH 15/17] feat(ENGKNOW-2892): Add gor query support for link files in gor server. --- .../s3/shared/S3SharedSourceProvider.java | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/drivers/src/main/java/org/gorpipe/s3/shared/S3SharedSourceProvider.java b/drivers/src/main/java/org/gorpipe/s3/shared/S3SharedSourceProvider.java index 76562ed5..1ba51b5e 100644 --- a/drivers/src/main/java/org/gorpipe/s3/shared/S3SharedSourceProvider.java +++ b/drivers/src/main/java/org/gorpipe/s3/shared/S3SharedSourceProvider.java @@ -106,7 +106,7 @@ public S3SharedSource resolveDataSource(SourceReference sourceReference) String project = Path.of(sourceReference.getCommonRoot()).getFileName().toString(); String bucket = sharedCreds.getLookupKey(); - String s3SecurityContext = createS3SecurityContext(sharedCreds); + String s3SecurityContext = createS3Credentials(sharedCreds).addToSecurityContext(sourceReference.getSecurityContext()); String relativePath = getRelativePath(sourceReference.getUrl()); SourceReference s3SourceReference = createS3SourceReference(sourceReference, project, bucket, s3SecurityContext); @@ -159,14 +159,18 @@ private Credentials getHighestPriorityCredential(List credsList, St return bestMatch; } - private String createS3SecurityContext(Credentials sharedCreds) { - BundledCredentials bundledCredentials = new BundledCredentials.Builder().addCredentials( + /** + * Creates a security context that add credentials for s3 direct access (based on the s3data creds) + * @param sharedCreds incoming shared credentials + * @return shared credentials with s3 creds added + */ + private BundledCredentials createS3Credentials(Credentials sharedCreds) { + BundledCredentials.Builder builder = new BundledCredentials.Builder().addCredentials( new Credentials("s3", sharedCreds.getLookupKey(), sharedCreds.getOwnerType(), sharedCreds.getOwnerId(), sharedCreds.expires(), sharedCreds.isUserDefault(), - (Map) sharedCreds.toMap().get("credential_attributes"))) - .build(); - String securityContext = bundledCredentials.addToSecurityContext(""); - return securityContext; + (Map) sharedCreds.toMap().get("credential_attributes"))); + + return builder.build(); } private SourceReference createS3SourceReference(SourceReference sourceReference, String project, String bucket, String securityContext) { From e095b5b3a5289de0b308d7a8f805cf4736545941 Mon Sep 17 00:00:00 2001 From: Gisli Magnusson Date: Wed, 3 Dec 2025 20:56:11 +0000 Subject: [PATCH 16/17] feat(ENGKNOW-2892): Add gor query support for link files in gor server. --- .../s3/shared/S3SharedSourceProvider.java | 37 ++++++++++--------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/drivers/src/main/java/org/gorpipe/s3/shared/S3SharedSourceProvider.java b/drivers/src/main/java/org/gorpipe/s3/shared/S3SharedSourceProvider.java index 1ba51b5e..3d917bd2 100644 --- a/drivers/src/main/java/org/gorpipe/s3/shared/S3SharedSourceProvider.java +++ b/drivers/src/main/java/org/gorpipe/s3/shared/S3SharedSourceProvider.java @@ -97,18 +97,19 @@ public S3SharedSource resolveDataSource(SourceReference sourceReference) S3SharedSource source = null; - Credentials sharedCreds = getS3DataCredentials(getService(), sourceReference.getSecurityContext()); + BundledCredentials bundledCreds = BundledCredentials.fromSecurityContext(sourceReference.securityContext); + Credentials sharedCreds = getS3DataCredentials(getService(), bundledCreds ); if (sharedCreds == null) { log.warn(String.format("No credentials found for %s. Returning emtpy source.", getService())); return source; } - String project = Path.of(sourceReference.getCommonRoot()).getFileName().toString(); - String bucket = sharedCreds.getLookupKey(); - String s3SecurityContext = createS3Credentials(sharedCreds).addToSecurityContext(sourceReference.getSecurityContext()); + String s3SecurityContext = createSecurityContext(createS3CredsFromShared(sharedCreds), bundledCreds); String relativePath = getRelativePath(sourceReference.getUrl()); + String project = Path.of(sourceReference.getCommonRoot()).getFileName().toString(); + String bucket = sharedCreds.getLookupKey(); SourceReference s3SourceReference = createS3SourceReference(sourceReference, project, bucket, s3SecurityContext); S3Client client = getClient(s3SecurityContext, bucket); @@ -135,8 +136,7 @@ protected String findSharedSourceLinkContent(S3SharedSource source) { } } - private Credentials getS3DataCredentials(String service, String securityContext) { - BundledCredentials bundledCreds = BundledCredentials.fromSecurityContext(securityContext); + private Credentials getS3DataCredentials(String service, BundledCredentials bundledCreds) { List credsList = bundledCreds.getCredentialsForService(service); return getHighestPriorityCredential(credsList, service); } @@ -159,18 +159,21 @@ private Credentials getHighestPriorityCredential(List credsList, St return bestMatch; } - /** - * Creates a security context that add credentials for s3 direct access (based on the s3data creds) - * @param sharedCreds incoming shared credentials - * @return shared credentials with s3 creds added - */ - private BundledCredentials createS3Credentials(Credentials sharedCreds) { - BundledCredentials.Builder builder = new BundledCredentials.Builder().addCredentials( - new Credentials("s3", sharedCreds.getLookupKey(), sharedCreds.getOwnerType(), - sharedCreds.getOwnerId(), sharedCreds.expires(), sharedCreds.isUserDefault(), - (Map) sharedCreds.toMap().get("credential_attributes"))); + private String createSecurityContext(Credentials s3Creds, BundledCredentials bundledCreds) { + BundledCredentials.Builder builder = new BundledCredentials.Builder().addCredentials(s3Creds); + for (String services : bundledCreds.services()) { + for (Credentials cred : bundledCreds.getCredentialsForService(services)) { + builder.addCredentials(cred); + } + } + + return builder.build().addToSecurityContext(""); + } - return builder.build(); + private Credentials createS3CredsFromShared(Credentials sharedCreds) { + return new Credentials("s3", sharedCreds.getLookupKey(), sharedCreds.getOwnerType(), + sharedCreds.getOwnerId(), sharedCreds.expires(), sharedCreds.isUserDefault(), + (Map) sharedCreds.toMap().get("credential_attributes")); } private SourceReference createS3SourceReference(SourceReference sourceReference, String project, String bucket, String securityContext) { From 636fab3b44d86e194df07fdff3f00d3d15cac7b1 Mon Sep 17 00:00:00 2001 From: Gisli Magnusson Date: Wed, 3 Dec 2025 22:36:56 +0000 Subject: [PATCH 17/17] feat(ENGKNOW-2892): Add gor query support for link files in gor server. --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5c19bcfb..6b2aa47d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -142,7 +142,7 @@ jobs: files: '**/TEST-*.xml' publishSnapshot: - #if: ${{ github.ref == 'refs/heads/main' }} + if: ${{ github.ref == 'refs/heads/main' }} needs: [test, slowTest, integrationTest] runs-on: ubuntu-latest steps: