diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5c19bcfb..6b2aa47d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -142,7 +142,7 @@ jobs: files: '**/TEST-*.xml' publishSnapshot: - #if: ${{ github.ref == 'refs/heads/main' }} + if: ${{ github.ref == 'refs/heads/main' }} needs: [test, slowTest, integrationTest] runs-on: ubuntu-latest steps: diff --git a/gortools/src/main/java/org/gorpipe/querydialogs/Dialog.java b/gortools/src/main/java/org/gorpipe/querydialogs/Dialog.java index b336a999..aca6b530 100644 --- a/gortools/src/main/java/org/gorpipe/querydialogs/Dialog.java +++ b/gortools/src/main/java/org/gorpipe/querydialogs/Dialog.java @@ -28,7 +28,6 @@ import freemarker.core.ParseException; import freemarker.template.*; import org.gorpipe.exceptions.GorResourceException; -import org.gorpipe.gor.model.Constants; import org.gorpipe.gor.model.FileReader; import org.gorpipe.gor.model.QueryEvaluator; import org.gorpipe.querydialogs.templating.DialogArgumentWrapper; @@ -583,13 +582,7 @@ public String toString() { } private void setConfig() { - if (Constants.isSet()) { - final String constantsProjectName = Constants.get().projectName(); - if (!constantsProjectName.equals(projectName)) { - initializeTemplateConfig(fileResolver, queryEval); - projectName = constantsProjectName; - } - } else if (TEMPLATE_CONFIG == null) { + if (TEMPLATE_CONFIG == null) { initializeTemplateConfig(fileResolver, queryEval); } } diff --git a/gortools/src/main/scala/gorsat/InputSources/Sql.scala b/gortools/src/main/scala/gorsat/InputSources/Sql.scala index 869d05d2..3390e27f 100644 --- a/gortools/src/main/scala/gorsat/InputSources/Sql.scala +++ b/gortools/src/main/scala/gorsat/InputSources/Sql.scala @@ -28,7 +28,7 @@ import gorsat.Utilities.AnalysisUtilities import gorsat.process.GorJavaUtilities import org.gorpipe.gor.model.GorOptions import org.gorpipe.gor.session.{GorContext, GorSession} -import org.gorpipe.gor.util.{CommandSubstitutions, SqlReplacer} +import org.gorpipe.gor.util.SqlReplacer import java.util @@ -55,12 +55,12 @@ object Sql { val database = if (hasOption(args, "-db")) stringValueOfOption(args, "-db") else null val map = new util.HashMap[String, Object]() - map.put(CommandSubstitutions.KEY_CHROM, range.chromosome) - map.put(CommandSubstitutions.KEY_BPSTART, range.start.toString) - map.put(CommandSubstitutions.KEY_BPSTOP, range.stop.toString) - map.put(CommandSubstitutions.KEY_DATABASE, database) - map.put(CommandSubstitutions.KEY_TAGS, tags ) - CommandSubstitutions.updateMapWithProjectInfo(session, map) + map.put(SqlReplacer.KEY_CHROM, range.chromosome) + map.put(SqlReplacer.KEY_BPSTART, range.start.toString) + map.put(SqlReplacer.KEY_BPSTOP, range.stop.toString) + map.put(SqlReplacer.KEY_DATABASE, database) + map.put(SqlReplacer.KEY_TAGS, tags ) + SqlReplacer.updateMapFromSecurityContext(session.getProjectContext.getFileReader.getSecurityContext, map) val iteratorSource = GorJavaUtilities.getDbIteratorSource(myCommand, map, database, !isNorContext, false) diff --git a/gortools/src/main/scala/gorsat/process/PipeInstance.scala b/gortools/src/main/scala/gorsat/process/PipeInstance.scala index 6301b5ff..602c6a7a 100644 --- a/gortools/src/main/scala/gorsat/process/PipeInstance.scala +++ b/gortools/src/main/scala/gorsat/process/PipeInstance.scala @@ -22,8 +22,6 @@ package gorsat.process -import java.io.File -import java.nio.file.{Files, Paths} import java.util import java.util.Optional import gorsat.Analysis._ @@ -117,6 +115,9 @@ object PipeInstance { */ class PipeInstance(context: GorContext, outputValidateOrder: Boolean = false) extends gorsatGorIterator(context) { + val DO_GLOBAL_QUERY_CONST_REPLACEMENTS: Boolean = System.getProperty("gor.query.const.replacements", "false").toBoolean + + // Define second constructor, needed for Java. def this(context: GorContext) = { this(context, false) @@ -259,6 +260,9 @@ class PipeInstance(context: GorContext, outputValidateOrder: Boolean = false) ex thePipeStep = PlaceHolder() var argString = CommandParseUtilities.removeComments(inputQuery) + if (DO_GLOBAL_QUERY_CONST_REPLACEMENTS) { + argString = CommandSubstitutions.projectReplacement(argString, context.getSession) + } val gorCommands = CommandParseUtilities.quoteSafeSplitAndTrim(argString, ';') // In case this is a command line script if (fileSignature || virtualFile != null || ScriptParsers.isScript(gorCommands)) { diff --git a/gortools/src/test/java/gorsat/UTestSQLInputSource.java b/gortools/src/test/java/gorsat/UTestSQLInputSource.java index ac5d9d2a..649ec791 100644 --- a/gortools/src/test/java/gorsat/UTestSQLInputSource.java +++ b/gortools/src/test/java/gorsat/UTestSQLInputSource.java @@ -26,6 +26,7 @@ import org.gorpipe.gor.model.DbConnection; import org.gorpipe.test.utils.FileTestUtils; import org.junit.*; +import org.junit.contrib.java.lang.system.RestoreSystemProperties; import org.junit.rules.TemporaryFolder; import java.io.File; @@ -47,6 +48,9 @@ public class UTestSQLInputSource { @Rule public TemporaryFolder workDir = new TemporaryFolder(); + @Rule + public final RestoreSystemProperties restoreSystemProperties = new RestoreSystemProperties(); + @BeforeClass public static void initDb() throws IOException, ClassNotFoundException, SQLException { rdaPaths = DatabaseHelper.createRdaDatabase(); @@ -97,8 +101,17 @@ public void testGorSqlWithSource() { } @Test - public void testNorSqlSourceWithProjectId() throws IOException, ClassNotFoundException { - String sqlcmd = "norsql {select * from rda.v_variant_annotations where project_id = #{project_id}}"; + public void testNorSqlSourceWithDbProjectId() { + String sqlcmd = "norsql {select * from rda.v_variant_annotations where project_id = #{project-id}}"; + String securityContext = "dbscope=project_id#int#10004|||extrastuff=other"; + var result = TestUtils.runGorPipe(sqlcmd, false, securityContext); + Assert.assertEquals(6, result.split("\n").length); + } + + @Test + public void testNorSqlSourceWithProjectId() { + System.setProperty("gor.query.const.replacements", "true"); + String sqlcmd = "norsql {select project_id,chromo,pos,pn,comment, #{project_id} x from rda.v_variant_annotations where project_id = '#{project_id}'}"; String securityContext = "dbscope=project_id#int#10004|||extrastuff=other"; var result = TestUtils.runGorPipe(sqlcmd, false, securityContext); Assert.assertEquals(6, result.split("\n").length); diff --git a/model/src/main/java/org/gorpipe/gor/driver/providers/rows/sources/sql/SqlSource.java b/model/src/main/java/org/gorpipe/gor/driver/providers/rows/sources/sql/SqlSource.java index 4c7756da..f18b97e4 100644 --- a/model/src/main/java/org/gorpipe/gor/driver/providers/rows/sources/sql/SqlSource.java +++ b/model/src/main/java/org/gorpipe/gor/driver/providers/rows/sources/sql/SqlSource.java @@ -33,7 +33,7 @@ import org.gorpipe.gor.model.GenomicIterator; import org.gorpipe.gor.model.StreamWrappedGenomicIterator; import org.gorpipe.gor.table.util.PathUtils; -import org.gorpipe.gor.util.CommandSubstitutions; +import org.gorpipe.gor.util.SqlReplacer; import java.util.HashMap; @@ -55,7 +55,7 @@ public GenomicIterator open() { sqlInfo = getInfoFromSql(sql); var constants = new HashMap(); - CommandSubstitutions.updateMapFromSecurityContext(sourceReference.securityContext, constants); + SqlReplacer.updateMapFromSecurityContext(sourceReference.securityContext, constants); var connectionCache = this.getSourceReference().getUrl().startsWith(LegacyDbSourceType.ProtocolName) ? DbConnection.systemConnections : diff --git a/model/src/main/java/org/gorpipe/gor/model/Constants.java b/model/src/main/java/org/gorpipe/gor/model/Constants.java deleted file mode 100644 index e1000733..00000000 --- a/model/src/main/java/org/gorpipe/gor/model/Constants.java +++ /dev/null @@ -1,112 +0,0 @@ -/* - * BEGIN_COPYRIGHT - * - * Copyright (C) 2011-2013 deCODE genetics Inc. - * Copyright (C) 2013-2019 WuXi NextCode Inc. - * All Rights Reserved. - * - * GORpipe is free software: you can redistribute it and/or modify - * it under the terms of the AFFERO GNU General Public License as published by - * the Free Software Foundation. - * - * GORpipe is distributed "AS-IS" AND WITHOUT ANY WARRANTY OF ANY KIND, - * INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, - * NON-INFRINGEMENT, OR FITNESS FOR A PARTICULAR PURPOSE. See - * the AFFERO GNU General Public License for the complete license terms. - * - * You should have received a copy of the AFFERO GNU General Public License - * along with GORpipe. If not, see - * - * END_COPYRIGHT - */ - -package org.gorpipe.gor.model; - - -/** - * Constants allow querying of an application wide constant - * - * @version $Id$ - */ -public abstract class Constants { - private static Constants constants = null; - - /** - * @return The system level constants - */ - public static Constants get() { - assert constants != null; - return constants; - } - - /** - * Set the active constants for the application - * - * @param consts The constants - */ - public static void set(Constants consts) { - constants = consts; - } - - /** - * @return true if constants have been set - */ - public static boolean isSet() { - return constants != null; - } - - /** - * Replace all occurances of the constants found the the specified text using default Constants, if set - * - * @param text - * @return The text after replacement - */ - public static String replaceConstantsInText(String text) { - return isSet() ? get().replaceConstantsInTextx(text) : text; - } - - /** - * Replace all occurances of the constants found the the specified text - * - * @param text - * @return The text after replacement - */ - public String replaceConstantsInTextx(String text) { - String t = text.replace("#{project-id}", String.valueOf(projectId())); - t = t.replace("#{project}", projectName()); - t = t.replace("#{user}", userName()); - return t; - } - - /** - * Check if project has changed. - * - * @param oldProjectName old project name - * @return true if project has changed, otherwise false - */ - public static boolean projectChanged(final String oldProjectName) { - boolean configChanged = false; - if (Constants.isSet()) { - final String constantsProjectName = Constants.get().projectName(); - if (!constantsProjectName.equals(oldProjectName)) { - configChanged = true; - } - } - return configChanged; - } - - /** - * @return The current project name - */ - abstract public String projectName(); - - /** - * @return The current loged in user - */ - abstract public String userName(); - - /** - * @return The current project id - */ - abstract public int projectId(); -} diff --git a/model/src/main/java/org/gorpipe/gor/model/DbNorIterator.java b/model/src/main/java/org/gorpipe/gor/model/DbNorIterator.java index 0c90695d..7573aa3e 100644 --- a/model/src/main/java/org/gorpipe/gor/model/DbNorIterator.java +++ b/model/src/main/java/org/gorpipe/gor/model/DbNorIterator.java @@ -29,8 +29,10 @@ import java.sql.*; import java.util.ArrayList; +import java.util.Arrays; import java.util.Iterator; import java.util.Map; +import java.util.stream.Collectors; /** @@ -39,6 +41,8 @@ */ public class DbNorIterator implements Iterator, AutoCloseable { + private static final org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(DbNorIterator.class); + private Connection conn = null; private PreparedStatement stmt = null; private ResultSet rs = null; @@ -57,9 +61,14 @@ public class DbNorIterator implements Iterator, AutoCloseable { */ public DbNorIterator(String content, Map constants, ConnectionPool pool) { + log.info("DB NOR query before replace: {}", content); // Replace scoping variables. Pair sqlWithParams = SqlReplacer.replaceConstants(content, constants); + log.info("Executing DB NOR query: {}", sqlWithParams.getFormer()); + log.info("With parameters: {}", sqlWithParams.getLatter() != null ? Arrays.asList(sqlWithParams.getLatter()) + .stream().map(o -> o != null ? o.toString() : "null").collect(Collectors.joining(", ")) : "Empty params"); + // Get db connection. try { conn = pool.getConnection(); diff --git a/model/src/main/java/org/gorpipe/gor/model/GorCommand.java b/model/src/main/java/org/gorpipe/gor/model/GorCommand.java index 8119edd6..5404d21c 100644 --- a/model/src/main/java/org/gorpipe/gor/model/GorCommand.java +++ b/model/src/main/java/org/gorpipe/gor/model/GorCommand.java @@ -158,19 +158,4 @@ private String removeComments(boolean allowQuotes) { return rc.toString(); } - /** - * @param cmd Gor Command - * @return command with user contstants - */ - public static String replaceClientConstants(String cmd) { - String t = cmd; - if (Constants.isSet()) { - t = t.replace("#{project}", Constants.get().projectName()); - t = t.replace("#{user}", Constants.get().userName()); - } - return t; - } - - - } diff --git a/model/src/main/java/org/gorpipe/gor/model/GorIterator.java b/model/src/main/java/org/gorpipe/gor/model/GorIterator.java index 4159abe1..6ac34949 100644 --- a/model/src/main/java/org/gorpipe/gor/model/GorIterator.java +++ b/model/src/main/java/org/gorpipe/gor/model/GorIterator.java @@ -238,7 +238,6 @@ public static String populateCommand(String cmd, String fName, String filtr, Str if (fName != null) cmd = cmd.replaceAll("\\$file", fName); if (range >= 0) cmd = cmd.replaceAll("\\$range", String.valueOf(range)); } - cmd = GorCommand.replaceClientConstants(cmd); return cmd; } diff --git a/model/src/main/java/org/gorpipe/gor/util/CommandSubstitutions.java b/model/src/main/java/org/gorpipe/gor/util/CommandSubstitutions.java index 70ff1fcf..7f48eff1 100644 --- a/model/src/main/java/org/gorpipe/gor/util/CommandSubstitutions.java +++ b/model/src/main/java/org/gorpipe/gor/util/CommandSubstitutions.java @@ -19,16 +19,8 @@ public class CommandSubstitutions { public static final String KEY_USER = "user"; public static final String KEY_PROJECT = "project"; public static final String KEY_PROJECT_ID = "project_id"; - public static final String KEY_DB_PROJECT_ID = "project-id"; public static final String KEY_ORGANIZATION_ID = "organization_id"; - public static final String KEY_DB_ORGANIZATION_ID = "organization-id"; - public static final String KEY_REQUEST_ID = "request_id"; - public static final String KEY_CHROM = "chrom"; - public static final String KEY_BPSTART = "bpstart"; - public static final String KEY_BPSTOP = "bpstop"; - public static final String KEY_TAGS = "tags"; - public static final String KEY_DATABASE = "database"; /** * Process a list of commands and apply filter and seek substitutions. @@ -257,11 +249,9 @@ public static Map updateMapFromSecurityContext(String securityCo var scopes = DbScope.parse(securityContext); for (var s : scopes) { if (s.getColumn().equalsIgnoreCase(KEY_PROJECT_ID)) { - map.put(KEY_DB_PROJECT_ID, s.getValue()); map.put(KEY_PROJECT_ID, s.getValue()); map.put("projectid", s.getValue()); } else if (s.getColumn().equalsIgnoreCase(KEY_ORGANIZATION_ID)) { - map.put(KEY_DB_ORGANIZATION_ID, s.getValue()); map.put(KEY_ORGANIZATION_ID, s.getValue()); } } diff --git a/model/src/main/java/org/gorpipe/gor/util/SqlReplacer.java b/model/src/main/java/org/gorpipe/gor/util/SqlReplacer.java index 5796a38a..bd431e7d 100644 --- a/model/src/main/java/org/gorpipe/gor/util/SqlReplacer.java +++ b/model/src/main/java/org/gorpipe/gor/util/SqlReplacer.java @@ -1,6 +1,7 @@ package org.gorpipe.gor.util; import org.gorpipe.exceptions.GorResourceException; +import org.gorpipe.gor.driver.providers.rows.sources.db.DbScope; import org.gorpipe.util.Pair; import java.util.Collection; @@ -8,9 +9,20 @@ import java.util.stream.Collectors; public class SqlReplacer { + private static final org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(SqlReplacer.class); final static String REGEX = "#\\{(\\S+?)\\}"; + // DB prepared statements + public static final String KEY_DB_PROJECT_ID = "project-id"; + public static final String KEY_DB_ORGANIZATION_ID = "organization-id"; + + public static final String KEY_CHROM = "chrom"; + public static final String KEY_BPSTART = "bpstart"; + public static final String KEY_BPSTOP = "bpstop"; + public static final String KEY_TAGS = "tags"; + public static final String KEY_DATABASE = "database"; + static Object[] replacementList(String sql, final Mapconstants) { var matcher = java.util.regex.Pattern.compile(REGEX).matcher(sql); @@ -63,4 +75,29 @@ public static Pair replaceConstants(final String sql, final Ma var newSql = SqlReplacer.replaceWithSqlParameter(sql, constants); return new Pair<>(newSql, replacements); } + + /** + * Update a map with project and organization IDs extracted from the security context string. + * + * @param securityContext The security context string containing project and organization information + * @param map The map to update with extracted values + * @return The updated map + */ + public static Map updateMapFromSecurityContext(String securityContext, Map map) { + if (map == null) return map; + log.info("Parsing security context for DB scopes: {}", securityContext); + var scopes = DbScope.parse(securityContext); + log.info("Extracted DB scopes from security context: {}", scopes.stream().map(s -> s.toString()).collect(Collectors.joining(", "))); + for (var s : scopes) { + if (s.getColumn().equalsIgnoreCase(CommandSubstitutions.KEY_PROJECT_ID)) { + map.put(KEY_DB_PROJECT_ID, s.getValue()); + log.info("Added project ID to DB scopes map: {}={}", KEY_DB_PROJECT_ID, s.getValue()); + } else if (s.getColumn().equalsIgnoreCase(CommandSubstitutions.KEY_ORGANIZATION_ID)) { + map.put(KEY_DB_ORGANIZATION_ID, s.getValue()); + } + } + + log.info("Final map of DB scopes: {}", map); + return map; + } }