Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import java.util.stream.Collectors;

public class BaseScriptExecutionEngine {

Map<String,ExecutionBlock> executionBlocks = new ConcurrentHashMap<>();
Map<String,String> aliases = new ConcurrentHashMap<>();
Map<String,String> fileSignatureMap = new ConcurrentHashMap<>();
Expand Down Expand Up @@ -142,14 +143,17 @@ public String getFileSignatureAndUpdateSignatureMap(GorSession session, String c
var signatureKey = AnalysisUtilities.getSignatureFromSignatureCommand(session, commandToExecute);
var fileListKey = String.join(" ", usedFiles);

if (usedFiles.stream().anyMatch(DataUtil::isYml) || MacroUtilities.containsWriteCommand(commandToExecute)) {
boolean SIDE_EFFECTS_FORCE_RUN = Boolean.parseBoolean(System.getProperty("gor.gorpipe.sideeffects.force_run", "false"));
if (usedFiles.stream().anyMatch(DataUtil::isYml)
|| (SIDE_EFFECTS_FORCE_RUN && MacroUtilities.containsWriteCommand(commandToExecute))) {
// Cases were we always want to run the query:
// 1. if any of the files is a template expansion, we treat this as if non-deterministic. We do not
// model how expansion might depend on files and parameters.
// We could expand the YML file and then signature that query according to its usedFiles etc.,
// but that is not too different from forcing a cache miss at this stage by using a never-reused
// signature value. We don't save it for reuse.
// 2. if the command contains a write command, we always want to run it for the sideeffect of writing.
// 2. if the command contains a write command, we want to run it for the sideeffect of writing
// if gor.gorpipe.sideeffects.force_run is sett.
fileSignature = StringUtilities.createMD5(System.nanoTime() + fileListKey + signatureKey);
} else {
fileSignature = fileSignatureMap.computeIfAbsent(
Expand Down
7 changes: 7 additions & 0 deletions gortools/src/test/java/gorsat/UTestGorWriteExplicit.java
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import org.gorpipe.gor.table.dictionary.gor.GorDictionaryTableMeta;
import org.junit.*;
import org.junit.contrib.java.lang.system.RestoreSystemProperties;
import org.junit.rules.TemporaryFolder;

import java.io.IOException;
Expand All @@ -15,6 +16,10 @@


public class UTestGorWriteExplicit {

@Rule
public final RestoreSystemProperties restoreSystemProperties = new RestoreSystemProperties();

public static final String WRONG_RESULT = "Wrong results in write folder";
static final String GROUP_CHROM_COUNT = " | group chrom -count";

Expand Down Expand Up @@ -544,6 +549,8 @@ public void testGordFolder() {

@Test
public void testCreateWriteSignature() throws IOException {
System.setProperty("gor.gorpipe.sideeffects.force_run", "true");

Path path = workDirPath.resolve("gorfile.gorz");
TestUtils.runGorPipe(
"create a = gor -p chr21 ../tests/data/gor/genes.gor | write " + path + "; " +
Expand Down
Loading