fix(ENGKNOW-2760): norsql add project id support like we have for db links

.github/workflows/build.yml

@@ -142,7 +142,7 @@ jobs:
           files: '**/TEST-*.xml'
 
   publishSnapshot:
-    #if: ${{ github.ref == 'refs/heads/main' }}
+    if: ${{ github.ref == 'refs/heads/main' }}
     needs: [test, slowTest, integrationTest]
     runs-on: ubuntu-latest
     steps:

gortools/src/main/java/org/gorpipe/querydialogs/Dialog.java

@@ -28,7 +28,6 @@
 import freemarker.core.ParseException;
 import freemarker.template.*;
 import org.gorpipe.exceptions.GorResourceException;
-import org.gorpipe.gor.model.Constants;
 import org.gorpipe.gor.model.FileReader;
 import org.gorpipe.gor.model.QueryEvaluator;
 import org.gorpipe.querydialogs.templating.DialogArgumentWrapper;
@@ -583,13 +582,7 @@ public String toString() {
     }
 
     private void setConfig() {
-        if (Constants.isSet()) {
-            final String constantsProjectName = Constants.get().projectName();
-            if (!constantsProjectName.equals(projectName)) {
-                initializeTemplateConfig(fileResolver, queryEval);
-                projectName = constantsProjectName;
-            }
-        } else if (TEMPLATE_CONFIG == null) {
+        if (TEMPLATE_CONFIG == null) {
             initializeTemplateConfig(fileResolver, queryEval);
         }
     }

gortools/src/main/scala/gorsat/InputSources/Sql.scala

@@ -28,7 +28,7 @@ import gorsat.Utilities.AnalysisUtilities
 import gorsat.process.GorJavaUtilities
 import org.gorpipe.gor.model.GorOptions
 import org.gorpipe.gor.session.{GorContext, GorSession}
-import org.gorpipe.gor.util.{CommandSubstitutions, SqlReplacer}
+import org.gorpipe.gor.util.SqlReplacer
 
 import java.util
 
@@ -55,12 +55,12 @@ object Sql {
     val database = if (hasOption(args, "-db")) stringValueOfOption(args, "-db") else null
 
     val map = new util.HashMap[String, Object]()
-    map.put(CommandSubstitutions.KEY_CHROM, range.chromosome)
-    map.put(CommandSubstitutions.KEY_BPSTART, range.start.toString)
-    map.put(CommandSubstitutions.KEY_BPSTOP, range.stop.toString)
-    map.put(CommandSubstitutions.KEY_DATABASE, database)
-    map.put(CommandSubstitutions.KEY_TAGS, tags )
-    CommandSubstitutions.updateMapWithProjectInfo(session, map)
+    map.put(SqlReplacer.KEY_CHROM, range.chromosome)
+    map.put(SqlReplacer.KEY_BPSTART, range.start.toString)
+    map.put(SqlReplacer.KEY_BPSTOP, range.stop.toString)
+    map.put(SqlReplacer.KEY_DATABASE, database)
+    map.put(SqlReplacer.KEY_TAGS, tags )
+    SqlReplacer.updateMapFromSecurityContext(session.getProjectContext.getFileReader.getSecurityContext, map)
 
     val iteratorSource = GorJavaUtilities.getDbIteratorSource(myCommand, map, database, !isNorContext, false)
 

gortools/src/main/scala/gorsat/process/PipeInstance.scala

@@ -22,8 +22,6 @@
 
 package gorsat.process
 
-import java.io.File
-import java.nio.file.{Files, Paths}
 import java.util
 import java.util.Optional
 import gorsat.Analysis._
@@ -117,6 +115,9 @@ object PipeInstance {
   */
 class PipeInstance(context: GorContext, outputValidateOrder: Boolean = false) extends gorsatGorIterator(context) {
 
+  val DO_GLOBAL_QUERY_CONST_REPLACEMENTS: Boolean = System.getProperty("gor.query.const.replacements", "false").toBoolean
+
+
   // Define second constructor, needed for Java.
   def this(context: GorContext) = {
     this(context, false)
@@ -259,6 +260,9 @@ class PipeInstance(context: GorContext, outputValidateOrder: Boolean = false) ex
     thePipeStep = PlaceHolder()
 
     var argString = CommandParseUtilities.removeComments(inputQuery)
+    if (DO_GLOBAL_QUERY_CONST_REPLACEMENTS) {
+      argString = CommandSubstitutions.projectReplacement(argString, context.getSession)
+    }
     val gorCommands = CommandParseUtilities.quoteSafeSplitAndTrim(argString, ';') // In case this is a command line script
 
     if (fileSignature || virtualFile != null || ScriptParsers.isScript(gorCommands)) {

gortools/src/test/java/gorsat/UTestSQLInputSource.java

@@ -26,6 +26,7 @@
 import org.gorpipe.gor.model.DbConnection;
 import org.gorpipe.test.utils.FileTestUtils;
 import org.junit.*;
+import org.junit.contrib.java.lang.system.RestoreSystemProperties;
 import org.junit.rules.TemporaryFolder;
 
 import java.io.File;
@@ -47,6 +48,9 @@ public class UTestSQLInputSource {
     @Rule
     public TemporaryFolder workDir = new TemporaryFolder();
 
+    @Rule
+    public final RestoreSystemProperties restoreSystemProperties = new RestoreSystemProperties();
+
     @BeforeClass
     public static void initDb() throws IOException, ClassNotFoundException, SQLException {
         rdaPaths = DatabaseHelper.createRdaDatabase();
@@ -97,8 +101,17 @@ public void testGorSqlWithSource() {
     }
 
     @Test
-    public void testNorSqlSourceWithProjectId() throws IOException, ClassNotFoundException {
-        String sqlcmd = "norsql {select * from rda.v_variant_annotations where project_id = #{project_id}}";
+    public void testNorSqlSourceWithDbProjectId() {
+        String sqlcmd = "norsql {select * from rda.v_variant_annotations where project_id = #{project-id}}";
+        String securityContext = "dbscope=project_id#int#10004|||extrastuff=other";
+        var result = TestUtils.runGorPipe(sqlcmd, false, securityContext);
+        Assert.assertEquals(6, result.split("\n").length);
+    }
+
+    @Test
+    public void testNorSqlSourceWithProjectId() {
+        System.setProperty("gor.query.const.replacements", "true");
+        String sqlcmd = "norsql {select project_id,chromo,pos,pn,comment, #{project_id} x from rda.v_variant_annotations where project_id = '#{project_id}'}";
         String securityContext = "dbscope=project_id#int#10004|||extrastuff=other";
         var result = TestUtils.runGorPipe(sqlcmd, false, securityContext);
         Assert.assertEquals(6, result.split("\n").length);

model/src/main/java/org/gorpipe/gor/driver/providers/rows/sources/sql/SqlSource.java

@@ -33,7 +33,7 @@
 import org.gorpipe.gor.model.GenomicIterator;
 import org.gorpipe.gor.model.StreamWrappedGenomicIterator;
 import org.gorpipe.gor.table.util.PathUtils;
-import org.gorpipe.gor.util.CommandSubstitutions;
+import org.gorpipe.gor.util.SqlReplacer;
 
 import java.util.HashMap;
 
@@ -55,7 +55,7 @@ public GenomicIterator open() {
         sqlInfo = getInfoFromSql(sql);
 
         var constants = new HashMap<String, Object>();
-        CommandSubstitutions.updateMapFromSecurityContext(sourceReference.securityContext, constants);
+        SqlReplacer.updateMapFromSecurityContext(sourceReference.securityContext, constants);
 
         var connectionCache = this.getSourceReference().getUrl().startsWith(LegacyDbSourceType.ProtocolName) ?
                 DbConnection.systemConnections :

model/src/main/java/org/gorpipe/gor/model/DbNorIterator.java

@@ -29,8 +29,10 @@
 
 import java.sql.*;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Iterator;
 import java.util.Map;
+import java.util.stream.Collectors;
 
 
 /**
@@ -39,6 +41,8 @@
  */
 public class DbNorIterator implements Iterator<String>, AutoCloseable {
 
+    private static final org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(DbNorIterator.class);
+
     private Connection conn = null;
     private PreparedStatement stmt = null;
     private ResultSet rs = null;
@@ -57,9 +61,14 @@ public class DbNorIterator implements Iterator<String>, AutoCloseable {
      */
     public DbNorIterator(String content, Map<String, Object> constants, ConnectionPool pool) {
 
+        log.info("DB NOR query before replace: {}", content);
         // Replace scoping variables.
         Pair<String, Object[]> sqlWithParams = SqlReplacer.replaceConstants(content, constants);
 
+        log.info("Executing DB NOR query: {}", sqlWithParams.getFormer());
+        log.info("With parameters: {}", sqlWithParams.getLatter() != null ? Arrays.asList(sqlWithParams.getLatter())
+                .stream().map(o -> o != null ? o.toString() : "null").collect(Collectors.joining(", ")) : "Empty params");
+
         // Get db connection.
         try {
             conn = pool.getConnection();

model/src/main/java/org/gorpipe/gor/model/GorCommand.java

@@ -158,19 +158,4 @@ private String removeComments(boolean allowQuotes) {
         return rc.toString();
     }
 
-    /**
-     * @param cmd Gor Command
-     * @return command with user contstants
-     */
-    public static String replaceClientConstants(String cmd) {
-        String t = cmd;
-        if (Constants.isSet()) {
-            t = t.replace("#{project}", Constants.get().projectName());
-            t = t.replace("#{user}", Constants.get().userName());
-        }
-        return t;
-    }
-
-
-
 }

model/src/main/java/org/gorpipe/gor/model/GorIterator.java

@@ -238,7 +238,6 @@ public static String populateCommand(String cmd, String fName, String filtr, Str
             if (fName != null) cmd = cmd.replaceAll("\\$file", fName);
             if (range >= 0) cmd = cmd.replaceAll("\\$range", String.valueOf(range));
         }
-        cmd = GorCommand.replaceClientConstants(cmd);
         return cmd;
     }
 

model/src/main/java/org/gorpipe/gor/util/CommandSubstitutions.java

@@ -19,16 +19,8 @@ public class CommandSubstitutions {
     public static final String KEY_USER = "user";
     public static final String KEY_PROJECT = "project";
     public static final String KEY_PROJECT_ID = "project_id";
-    public static final String KEY_DB_PROJECT_ID = "project-id";
     public static final String KEY_ORGANIZATION_ID = "organization_id";
-    public static final String KEY_DB_ORGANIZATION_ID = "organization-id";
-
     public static final String KEY_REQUEST_ID = "request_id";
-    public static final String KEY_CHROM = "chrom";
-    public static final String KEY_BPSTART = "bpstart";
-    public static final String KEY_BPSTOP = "bpstop";
-    public static final String KEY_TAGS = "tags";
-    public static final String KEY_DATABASE = "database";
 
     /**
      * Process a list of commands and apply filter and seek substitutions.
@@ -257,11 +249,9 @@ public static Map<String, Object> updateMapFromSecurityContext(String securityCo
         var scopes = DbScope.parse(securityContext);
         for (var s : scopes) {
             if (s.getColumn().equalsIgnoreCase(KEY_PROJECT_ID)) {
-                map.put(KEY_DB_PROJECT_ID, s.getValue());
                 map.put(KEY_PROJECT_ID, s.getValue());
                 map.put("projectid", s.getValue());
             } else if (s.getColumn().equalsIgnoreCase(KEY_ORGANIZATION_ID)) {
-                map.put(KEY_DB_ORGANIZATION_ID, s.getValue());
                 map.put(KEY_ORGANIZATION_ID, s.getValue());
             }
         }