From 10ee6df2711587a85bd9da08c65ffe63f89e3402 Mon Sep 17 00:00:00 2001 From: Gisli Magnusson Date: Thu, 19 Jun 2025 11:44:40 +0000 Subject: [PATCH 1/4] fix(ENGKNOW-2515): Write does NOT trigger rerun. --- .../main/java/gorsat/Script/BaseScriptExecutionEngine.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/gortools/src/main/java/gorsat/Script/BaseScriptExecutionEngine.java b/gortools/src/main/java/gorsat/Script/BaseScriptExecutionEngine.java index dadac3fb..7525f177 100644 --- a/gortools/src/main/java/gorsat/Script/BaseScriptExecutionEngine.java +++ b/gortools/src/main/java/gorsat/Script/BaseScriptExecutionEngine.java @@ -22,6 +22,9 @@ import java.util.stream.Collectors; public class BaseScriptExecutionEngine { + + public static final boolean SIDE_EFFECTS_FORCE_RUN = Boolean.parseBoolean(System.getProperty("gor.sideeffects.force.run", "false")); + Map executionBlocks = new ConcurrentHashMap<>(); Map aliases = new ConcurrentHashMap<>(); Map fileSignatureMap = new ConcurrentHashMap<>(); @@ -142,7 +145,8 @@ public String getFileSignatureAndUpdateSignatureMap(GorSession session, String c var signatureKey = AnalysisUtilities.getSignatureFromSignatureCommand(session, commandToExecute); var fileListKey = String.join(" ", usedFiles); - if (usedFiles.stream().anyMatch(DataUtil::isYml) || MacroUtilities.containsWriteCommand(commandToExecute)) { + if (usedFiles.stream().anyMatch(DataUtil::isYml) + || (SIDE_EFFECTS_FORCE_RUN && MacroUtilities.containsWriteCommand(commandToExecute))) { // Cases were we always want to run the query: // 1. if any of the files is a template expansion, we treat this as if non-deterministic. We do not // model how expansion might depend on files and parameters. From 56ac7deca0cd067566a0b0b78bf31d20acc2731b Mon Sep 17 00:00:00 2001 From: Gisli Magnusson Date: Thu, 19 Jun 2025 12:00:48 +0000 Subject: [PATCH 2/4] fix(ENGKNOW-2515): Write does NOT trigger rerun. --- .../main/java/gorsat/Script/BaseScriptExecutionEngine.java | 2 +- gortools/src/test/java/gorsat/UTestGorWriteExplicit.java | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/gortools/src/main/java/gorsat/Script/BaseScriptExecutionEngine.java b/gortools/src/main/java/gorsat/Script/BaseScriptExecutionEngine.java index 7525f177..637ec135 100644 --- a/gortools/src/main/java/gorsat/Script/BaseScriptExecutionEngine.java +++ b/gortools/src/main/java/gorsat/Script/BaseScriptExecutionEngine.java @@ -23,7 +23,7 @@ public class BaseScriptExecutionEngine { - public static final boolean SIDE_EFFECTS_FORCE_RUN = Boolean.parseBoolean(System.getProperty("gor.sideeffects.force.run", "false")); + public static final boolean SIDE_EFFECTS_FORCE_RUN = Boolean.parseBoolean(System.getProperty("gor.gorpipe.sideeffects.force_run", "false")); Map executionBlocks = new ConcurrentHashMap<>(); Map aliases = new ConcurrentHashMap<>(); diff --git a/gortools/src/test/java/gorsat/UTestGorWriteExplicit.java b/gortools/src/test/java/gorsat/UTestGorWriteExplicit.java index 77775a83..21ff4b23 100644 --- a/gortools/src/test/java/gorsat/UTestGorWriteExplicit.java +++ b/gortools/src/test/java/gorsat/UTestGorWriteExplicit.java @@ -2,6 +2,7 @@ import org.gorpipe.gor.table.dictionary.gor.GorDictionaryTableMeta; import org.junit.*; +import org.junit.contrib.java.lang.system.RestoreSystemProperties; import org.junit.rules.TemporaryFolder; import java.io.IOException; @@ -15,6 +16,10 @@ public class UTestGorWriteExplicit { + + @Rule + public final RestoreSystemProperties restoreSystemProperties = new RestoreSystemProperties(); + public static final String WRONG_RESULT = "Wrong results in write folder"; static final String GROUP_CHROM_COUNT = " | group chrom -count"; @@ -544,6 +549,8 @@ public void testGordFolder() { @Test public void testCreateWriteSignature() throws IOException { + System.setProperty("gor.gorpipe.sideeffects.force_run", "true"); + Path path = workDirPath.resolve("gorfile.gorz"); TestUtils.runGorPipe( "create a = gor -p chr21 ../tests/data/gor/genes.gor | write " + path + "; " + From f3ea70377aef7bcb394acfd88a09eb9ccea55622 Mon Sep 17 00:00:00 2001 From: Gisli Magnusson Date: Fri, 20 Jun 2025 11:01:40 +0000 Subject: [PATCH 3/4] fix(ENGKNOW-2515): Write does NOT trigger rerun. --- .../src/main/java/gorsat/Script/BaseScriptExecutionEngine.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/gortools/src/main/java/gorsat/Script/BaseScriptExecutionEngine.java b/gortools/src/main/java/gorsat/Script/BaseScriptExecutionEngine.java index 637ec135..95a9aadd 100644 --- a/gortools/src/main/java/gorsat/Script/BaseScriptExecutionEngine.java +++ b/gortools/src/main/java/gorsat/Script/BaseScriptExecutionEngine.java @@ -23,8 +23,6 @@ public class BaseScriptExecutionEngine { - public static final boolean SIDE_EFFECTS_FORCE_RUN = Boolean.parseBoolean(System.getProperty("gor.gorpipe.sideeffects.force_run", "false")); - Map executionBlocks = new ConcurrentHashMap<>(); Map aliases = new ConcurrentHashMap<>(); Map fileSignatureMap = new ConcurrentHashMap<>(); @@ -145,6 +143,7 @@ public String getFileSignatureAndUpdateSignatureMap(GorSession session, String c var signatureKey = AnalysisUtilities.getSignatureFromSignatureCommand(session, commandToExecute); var fileListKey = String.join(" ", usedFiles); + boolean SIDE_EFFECTS_FORCE_RUN = Boolean.parseBoolean(System.getProperty("gor.gorpipe.sideeffects.force_run", "false")); if (usedFiles.stream().anyMatch(DataUtil::isYml) || (SIDE_EFFECTS_FORCE_RUN && MacroUtilities.containsWriteCommand(commandToExecute))) { // Cases were we always want to run the query: From 0eab1c5ed07bad11114e0aead6fc576cd81c99ba Mon Sep 17 00:00:00 2001 From: Gisli Magnusson Date: Fri, 20 Jun 2025 11:02:33 +0000 Subject: [PATCH 4/4] fix(ENGKNOW-2515): Write does NOT trigger rerun. --- .../src/main/java/gorsat/Script/BaseScriptExecutionEngine.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gortools/src/main/java/gorsat/Script/BaseScriptExecutionEngine.java b/gortools/src/main/java/gorsat/Script/BaseScriptExecutionEngine.java index 95a9aadd..ccfe247d 100644 --- a/gortools/src/main/java/gorsat/Script/BaseScriptExecutionEngine.java +++ b/gortools/src/main/java/gorsat/Script/BaseScriptExecutionEngine.java @@ -152,7 +152,8 @@ public String getFileSignatureAndUpdateSignatureMap(GorSession session, String c // We could expand the YML file and then signature that query according to its usedFiles etc., // but that is not too different from forcing a cache miss at this stage by using a never-reused // signature value. We don't save it for reuse. - // 2. if the command contains a write command, we always want to run it for the sideeffect of writing. + // 2. if the command contains a write command, we want to run it for the sideeffect of writing + // if gor.gorpipe.sideeffects.force_run is sett. fileSignature = StringUtilities.createMD5(System.nanoTime() + fileListKey + signatureKey); } else { fileSignature = fileSignatureMap.computeIfAbsent(