From 1a3b9a2c9193afbbd07c156dbfeb9907ac5630ae Mon Sep 17 00:00:00 2001 From: labkey-matthewb Date: Mon, 15 Sep 2025 11:33:29 -0700 Subject: [PATCH 01/25] sql experiment --- query/build.gradle | 13 + .../query/controllers/QueryController.java | 1099 ++++++++++++++++- .../src/org/labkey/query/view/sourceQuery.jsp | 413 +++++-- query/webapp/query/QueryEditorPanel.js | 9 + 4 files changed, 1415 insertions(+), 119 deletions(-) diff --git a/query/build.gradle b/query/build.gradle index 33289e9b980..02aa43e8afe 100644 --- a/query/build.gradle +++ b/query/build.gradle @@ -27,6 +27,19 @@ dependencies { ) ) + BuildUtils.addExternalDependency( + project, + new ExternalDependency( + "com.google.genai:google-genai:1.15.0", + "GENAI", + "GENAI", + "https://google.com/", + "???", + "???", + "GenAI" + ) + ) + BuildUtils.addExternalDependency( project, new ExternalDependency( diff --git a/query/src/org/labkey/query/controllers/QueryController.java b/query/src/org/labkey/query/controllers/QueryController.java index 22de3dae615..83340488f35 100644 --- a/query/src/org/labkey/query/controllers/QueryController.java +++ b/query/src/org/labkey/query/controllers/QueryController.java @@ -19,6 +19,12 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.genai.types.FunctionDeclaration; +import com.google.genai.types.GenerateContentConfig; +import com.google.genai.types.Part; +import com.google.genai.types.Schema; +import com.google.genai.types.Tool; +import com.google.genai.types.Type; import jakarta.servlet.ServletException; import jakarta.servlet.http.HttpServletRequest; import jakarta.servlet.http.HttpServletResponse; @@ -131,6 +137,7 @@ import org.labkey.api.data.SqlSelector; import org.labkey.api.data.TSVWriter; import org.labkey.api.data.Table; +import org.labkey.api.data.TableDescription; import org.labkey.api.data.TableInfo; import org.labkey.api.data.TableSelector; import org.labkey.api.data.VirtualTable; @@ -198,6 +205,7 @@ import org.labkey.api.security.MutableSecurityPolicy; import org.labkey.api.security.RequiresAllOf; import org.labkey.api.security.RequiresAnyOf; +import org.labkey.api.security.RequiresLogin; import org.labkey.api.security.RequiresNoPermission; import org.labkey.api.security.RequiresPermission; import org.labkey.api.security.SecurityManager; @@ -234,6 +242,7 @@ import org.labkey.api.util.Pair; import org.labkey.api.util.ResponseHelper; import org.labkey.api.util.ReturnURLString; +import org.labkey.api.util.SessionHelper; import org.labkey.api.util.StringExpression; import org.labkey.api.util.TestContext; import org.labkey.api.util.URLHelper; @@ -315,6 +324,7 @@ import java.io.IOException; import java.io.OutputStream; import java.io.PrintWriter; +import java.lang.reflect.Method; import java.nio.file.Path; import java.sql.Connection; import java.sql.ResultSet; @@ -337,7 +347,14 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import com.google.common.collect.ImmutableList; +import com.google.genai.Chat; +import com.google.genai.Client; +import com.google.genai.types.Content; +import com.google.genai.types.GenerateContentResponse; + import static org.apache.commons.lang3.StringUtils.isBlank; +import static org.apache.commons.lang3.StringUtils.isNotBlank; import static org.apache.commons.lang3.StringUtils.trimToEmpty; import static org.labkey.api.action.ApiJsonWriter.CONTENT_TYPE_JSON; import static org.labkey.api.assay.AssayFileWriter.ensureUploadDirectory; @@ -2124,8 +2141,8 @@ protected void renderSheets(Workbook workbook) .setRenamedColumns(qf.getRenameColumnMap()); String sheetName = qf.getSheetName(); qv.configureExcelWriter(this, config); - String name = StringUtils.isNotBlank(sheetName)? sheetName : qv.getQueryDef().getName(); - name = StringUtils.isNotBlank(name)? name : StringUtils.isNotBlank(qv.getDataRegionName()) ? qv.getDataRegionName() : "Data"; + String name = isNotBlank(sheetName)? sheetName : qv.getQueryDef().getName(); + name = isNotBlank(name)? name : isNotBlank(qv.getDataRegionName()) ? qv.getDataRegionName() : "Data"; setSheetName(name); setAutoSize(true); renderNewSheet(workbook); @@ -6583,7 +6600,7 @@ public ApiResponse execute(final SetCheckForm form, BindException errors) throws { for (String id : ids) { - if (StringUtils.isNotBlank(id)) + if (isNotBlank(id)) selection.add(id); } } @@ -6657,7 +6674,7 @@ public ApiResponse execute(final SetCheckForm form, BindException errors) { for (String id : ids) { - if (StringUtils.isNotBlank(id)) + if (isNotBlank(id)) selection.add(id); } } @@ -6683,7 +6700,7 @@ public ApiResponse execute(final SetCheckForm form, BindException errors) { for (String id : ids) { - if (StringUtils.isNotBlank(id)) + if (isNotBlank(id)) selection.add(id); } } @@ -8743,4 +8760,1076 @@ private JSONArray getTestRows(String val) return rows; } } + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + public static class QueryWriterAction extends SimpleViewAction + { + @Override + public ModelAndView getView(Object o, BindException errors) throws Exception + { + return new JspView<>("/org/labkey/query/view/queryWriter.jsp", null, errors); + } + + @Override + public void addNavTrail(NavTree root) + { + + } + } + + + public static class PromptForm + { + String prompt; + String schemaName; + + public String getSchemaName() + { + return schemaName; + } + + public void setSchemaName(String schemaName) + { + this.schemaName = schemaName; + } + + public void setPrompt(String prompt) + { + this.prompt = prompt; + } + + public String getPrompt() + { + return this.prompt; + } + } + + + @RequiresPermission(ReadPermission.class) + @RequiresLogin + public static class QueryAgentAction extends ReadOnlyApiAction + { + String getSQLHelp() + { + // TODO use markdown translation instead of plain-text + return """ + This is documentation about LabKey SQL dialect: + + LabKey SQL + + LabKey SQL is a SQL dialect that supports (1) most standard SQL functionality and (2) provides extended functionality that is unique to LabKey, including: + + Security. Before execution, all SQL queries are checked against the user's security roles/permissions. + Lookup columns. Lookup columns use an intuitive syntax to access data in other tables to achieve what would normally require a JOIN statement. For example: "SomeTable.ForeignKey.FieldFromForeignTable" The special lookup column "Datasets" is injected into each study dataset and provides a syntax shortcut when joining the current dataset to another dataset that has compatible join keys. See example usage. + Cross-folder querying. Queries can be scoped to folders broader than the current folder and can draw from tables in folders other than the current folder. See Cross-Folder Queries. + Parameterized SQL statements. The PARAMETERS keyword lets you define parameters for a query. An associated API gives you control over the parameterized query from JavaScript code. See Parameterized SQL Queries. + Pivot tables. The PIVOT...BY and PIVOT...IN expressions provide a syntax for creating pivot tables. See Pivot Queries. + User-related functions. USERID() and ISMEMBEROF(groupid) let you control query visibility based on the user's group membership. + Ontology-related functions. (Premium Feature) Access preferred terms and ontology concepts from SQL queries. See Ontology SQL. + Lineage-related functions. (Premium Feature) Access ancestors and descendants of samples and data class entities. See Lineage SQL Queries. + Annotations. Override some column metadata using SQL annotations. See Use SQL Annotations. + + Reference Tables: + + Keywords + Constants + Operators + Operator Order of Precedence + Aggregate Functions - General + Aggregate Functions - PostgreSQL Only + SQL Functions - General + SQL Functions - PostgreSQL Specific + JSON and JSONB Operators and Functions + General Syntax + + Keywords + Keyword Description + AS Aliases can be explicitly named using the AS keyword. Note that the AS keyword is optional: the following select clauses both create an alias called "Name": + + SELECT LCASE(FirstName) AS Name + SELECT LCASE(FirstName) Name + + Implicit aliases are automatically generated for expressions in the SELECT list. In the query below, an output column named "Expression1" is automatically created for the expression "LCASE(FirstName)": + + SELECT LCASE(FirstName) + FROM PEOPLE + ASCENDING, ASC Return ORDER BY results in ascending value order. See the ORDER BY section for troubleshooting notes. + + ORDER BY Weight ASC + CAST(AS) CAST(R.d AS VARCHAR) + + Defined valid datatype keywords which can be used as cast/convert targets, and to what java.sql.Types name each keyword maps. Keywords are case-insensitive. + + BIGINT + BINARY + BIT + CHAR + DECIMAL + DATE + DOUBLE + FLOAT + GUID + INTEGER + LONGVARBINARY + LONGVARCHAR + NUMERIC + REAL + SMALLINT + TIME + TIMESTAMP + TINYINT + VARBINARY + VARCHAR + + Examples: + + CAST(TimeCreated AS DATE) + + CAST(WEEK(i.date) as INTEGER) as WeekOfYear, + + Precision and scale are supported when casting to NUMERIC. Example: + + CAST($Num AS NUMERIC(10,2)) + DESCENDING, DESC Return ORDER BY results in descending value order. See the ORDER BY section for troubleshooting notes. + + ORDER BY Weight DESC + DISTINCT Return distinct, non duplicate, values. + + SELECT DISTINCT Country + FROM Demographics + EXISTS Returns a Boolean value based on a subquery. Returns TRUE if at least one row is returned from the subquery. + + The following example returns any plasma samples which have been assayed with a score greater than 80%. Assume that ImmuneScores.Data.SpecimenId is a lookup field (aka foreign key) to Plasma.Name.; + + SELECT Plasma.Name + FROM Plasma + WHERE EXISTS + (SELECT * + FROM assay.General.ImmuneScores.Data + WHERE SpecimenId = Plasma.Name + AND ScorePercent > .8) + FALSE \s + FROM The FROM clause in LabKey SQL must contain at least one table. It can also contain JOINs to other tables. Commas are supported in the FROM clause: + + FROM TableA, TableB + WHERE TableA.x = TableB.x + + Nested joins are supported in the FROM clause: + + FROM TableA LEFT JOIN (TableB INNER JOIN TableC ON ...) ON... + + To refer to tables in LabKey folders other than the current folder, see Cross-Folder Queries. + GROUP BY Used with aggregate functions to group the results. Defines the "for each" or "per". The example below returns the number of records "for each" participant: + + SELECT ParticipantId, COUNT(Created) "Number of Records" + FROM "Physical Exam" + GROUP BY ParticipantId + HAVING Used with aggregate functions to limit the results. The following example returns participants with 10 or more records in the Physical Exam table: + + SELECT ParticipantId, COUNT(Created) "Number of Records" + FROM "Physical Exam" + GROUP BY ParticipantId + HAVING COUNT(Created) > 10 + + HAVING can be used without a GROUP BY clause, in which case all selected rows are treated as a single group for aggregation purposes. + JOIN, + RIGHT JOIN, + LEFT JOIN, + FULL JOIN, + CROSS JOIN Example: + + SELECT * + FROM "Physical Exam" + FULL JOIN "Lab Results" + ON "Physical Exam".ParticipantId = "Lab Results".ParticipantId + LIMIT Limits the number or records returned by the query. The following example returns the 10 most recent records: + + SELECT * + FROM "Physical Exam" + ORDER BY Created DESC LIMIT 10 + NULLIF(A,B) Returns NULL if A=B, otherwise returns A. + ORDER BY One option for sorting query results. It may produce unexpected results when dataregions or views also have sorting applied, or when using an expression in the ORDER BY clause, including an expression like table.columnName. If you can instead use a sort on the custom view or via, the API, those methods are preferred (see Troubleshooting note below). + + For best ORDER BY results, be sure to a) SELECT the columns on which you are sorting, b) sort on the SELECT column, not on an expression. To sort on an expression, include the expression in the SELECT (hidden if desired) and sort by the alias of the expression. For example: + + SELECT A, B, A+B AS C @hidden ... ORDER BY C + ...is preferable to: + SELECT A, B ... ORDER BY A+B + + Use ORDER BY with LIMIT to improve performance: + + SELECT ParticipantID, + Height_cm AS Height + FROM "Physical Exam" + ORDER BY Height DESC LIMIT 5 + + Troubleshooting: "Why is the ORDER BY clause not working as expected?" + + 1. Check to ensure you are sorting by a SELECT column (preferred) or an alias of an expression. Syntax like including the table name (i.e. ...ORDER BY table.columnName ASC) is an expression and should be aliased in the SELECT statement instead (i.e. SELECT table.columnName AS C ... ORDER BY C + + 2. When authoring queries in LabKey SQL, the query is typically processed as a subquery within a parent query. This parent query may apply it's own sorting overriding the ORDER BY clause in the subquery. This parent "view layer" provides default behavior like pagination, lookups, etc. but may also unexpectedly apply an additional sort. + + Two recommended solutions for more predictable sorting: + (A) Define the sort in the parent query using the grid view customizer. This may involve adding a new named view of that query to use as your parent query. + (B) Use the "sort" property in the selectRows API call. + PARAMETERS Queries can declare parameters using the PARAMETERS keyword. Default values data types are supported as shown below: + + PARAMETERS (X INTEGER DEFAULT 37) + SELECT * + FROM "Physical Exam" + WHERE Temp_C = X + + Parameter names will override any unqualified table column with the same name. Use a table qualification to disambiguate. In the example below, R.X refers to the column while X refers to the parameter: + + PARAMETERS(X INTEGER DEFAULT 5) + SELECT * + FROM Table R + WHERE R.X = X + + Supported data types for parameters are: BIGINT, BIT, CHAR, DECIMAL, DOUBLE, FLOAT, INTEGER, LONGVARCHAR, NUMERIC, REAL, SMALLINT, TIMESTAMP, TINYINT, VARCHAR + + Numeric parameters can include precision and scale: + + PARAMETERS($NUM NUMERIC(10,2)) + + Parameter values can be passed via JavaScript API calls to the query. For details see Parameterized SQL Queries. + PIVOT/PIVOT...BY/PIVOT...IN Re-visualize a table by rotating or "pivoting" a portion of it, essentially promoting cell data to column headers. See Pivot Queries for details and examples. + SELECT SELECT queries are the only type of query that can currently be written in LabKey SQL. Sub-selects are allowed both as an expression, and in the FROM clause. + + Aliases are automatically generated for expressions after SELECT. In the query below, an output column named "Expression1" is automatically generated for the expression "LCASE(FirstName)": + + SELECT LCASE(FirstName) FROM... + TRUE \s + UNION, UNION ALL The UNION clause is the same as standard SQL. LabKey SQL supports UNION in subqueries. + VALUES ... AS A subset of VALUES syntax is supported. Generate a "constant table" by providing a parenthesized list of expressions for each row in the table. The lists must all have the same number of elements and corresponding entries must have compatible data types. For example: + + VALUES (1, 'one'), (2, 'two'), (3, 'three') AS t; + You must provide the alias for the result ("AS t" in the above), aliasing column names is not supported. The column names will be 'column1', 'column2', etc. + WHERE Filter the results for certain values. Example: + + SELECT * + FROM "Physical Exam" + WHERE YEAR(Date) = 2010 + WITH + + Define a "common table expression" which functions like a subquery or inline view table. Especially useful for recursive queries. + + Usage Notes: If there are UNION clauses that do not reference the common table expression (CTE) itself, the server interprets them as normal UNIONs. The first subclause of a UNION may not reference the CTE. The CTE may only be referenced once in a FROM clause or JOIN clauses within the UNION. There may be multiple CTEs defined in the WITH. Each may reference the previous CTEs in the WITH. No column specifications are allowed in the WITH (as some SQL versions allow). + + Exception Behavior: Testing indicates that PostgreSQL does not provide an exception to LabKey Server for a non-ending, recursive CTE query. This can cause the LabKey Server to wait indefinitely for the query to complete. + + A non-recursive example: + + WITH AllDemo AS + ( + SELECT * + FROM "/Studies/Study A/".study.Demographics + UNION + SELECT * + FROM "/Studies/Study B/".study.Demographics + ) + SELECT ParticipantId from AllDemo + + A recursive example: In a table that holds parent/child information, this query returns all of the children and grandchildren (recursively down the generations), for a given "Source" parent. + + PARAMETERS + ( + Source VARCHAR DEFAULT NULL + ) + + WITH Derivations AS\s + (\s + -- Anchor Query. User enters a 'Source' parent\s + SELECT Item, Parent + FROM Items\s + WHERE Parent = Source + UNION ALL\s + + -- Recursive Query. Get the children, grandchildren, ... of the source parent + SELECT i.Item, i.Parent\s + FROM Items i INNER JOIN Derivations p\s + ON i.Parent = p.Item\s + )\s + SELECT * FROM Derivations + + + Constants + + The following constant values can be used in LabKey SQL queries. + Constant Description + CAST('Infinity' AS DOUBLE) Represents positive infinity. + CAST('-Infinity' AS DOUBLE) Represents negative infinity. + CAST('NaN' AS DOUBLE) Represents "Not a number". + TRUE Boolean value. + FALSE Boolean value. + + Operators + Operator Description + String Operators Note that strings are delimited with single quotes. Double quotes are used for column and table names containing spaces. + || String concatenation. For example: + + SELECT ParticipantId, + City || ', ' || State AS CityOfOrigin + FROM Demographics + + If any argument is null, the || operator will return a null string. To handle this, use COALESCE with an empty string as it's second argument, so that the other || arguments will be returned: + City || ', ' || COALESCE (State, '') + LIKE Pattern matching. The entire string must match the given pattern. Ex: LIKE 'W%'. + NOT LIKE Negative pattern matching. Will return values that do not match a given pattern. Ex: NOT LIKE 'W%' + Arithmetic Operators \s + + Add + - Subtract + * Multiply + / Divide + Comparison operators \s + = Equals + != Does not equal + <> Does not equal + > Is greater than + < Is less than + >= Is greater than or equal to + <= Is less than or equal to + IS NULL Is NULL + IS NOT NULL Is NOT NULL + BETWEEN Between two values. Values can be numbers, strings or dates. + IN Example: WHERE City IN ('Seattle', 'Portland') + NOT IN Example: WHERE City NOT IN ('Seattle', 'Portland') + Bitwise Operators \s + & Bitwise AND + | Bitwise OR + ^ Bitwise exclusive OR + Logical Operators \s + AND Logical AND + OR Logical OR + NOT Example: WHERE NOT Country='USA' + Operator Order of Precedence + Order of Precedence Operators + 1 - (unary) , + (unary), CASE + 2 *, / (multiplication, division) + 3 +, -, & (binary plus, binary minus) + 4 & (bitwise and) + 5 ^ (bitwise xor) + 6 | (bitwise or) + 7 || (concatenation) + 8 <, >, <=, >=, IN, NOT IN, BETWEEN, NOT BETWEEN, LIKE, NOT LIKE + 9 =, IS, IS NOT, <>, != + 10 NOT + 11 AND + 12 OR + Aggregate Functions - General + Function Description + COUNT The special syntax COUNT(*) is supported as of LabKey v9.2. + MIN Minimum + MAX Maximum + AVG Average + SUM Sum\s + GROUP_CONCAT An aggregate function, much like MAX, MIN, AVG, COUNT, etc. It can be used wherever the standard aggregate functions can be used, and is subject to the same grouping rules. It will return a string value which is comma-separated list of all of the values for that grouping. A custom separator, instead of the default comma, can be specified. Learn more here. The example below specifies a semi-colon as the separator: + + SELECT Participant, GROUP_CONCAT(DISTINCT Category, ';') AS CATEGORIES FROM SomeSchema.SomeTable + + To use a line-break as the separator, use the following: + + SELECT Participant, GROUP_CONCAT(DISTINCT Category, chr(10)) AS CATEGORIES FROM SomeSchema.SomeTable + stddev(expression) Standard deviation + stddev_pop(expression) Population standard deviation of the input values. + variance(expression) Historical alias for var_samp. + var_pop(expression) Population variance of the input values (square of the population standard deviation). + median(expression) The 50th percentile of the values submitted. + Aggregate Functions - PostgreSQL Only + Function Description + bool_and(expression) Aggregates boolean values. Returns true if all values are true and false if any are false. + bool_or(expression) Aggregates boolean values. Returns true if any values are true and false if all are false. + bit_and(expression) Returns the bitwise AND of all non-null input values, or null if none. + bit_or(expression) Returns the bitwise OR of all non-null input values, or null if none. + every(expression) Equivalent to bool_and(). Returns true if all values are true and false if any are false. + corr(Y,X) Correlation coefficient. + covar_pop(Y,X) Population covariance. + covar_samp(Y,X) Sample covariance. + regr_avgx(Y,X) Average of the independent variable: (SUM(X)/N). + regr_avgy(Y,X) Average of the dependent variable: (SUM(Y)/N). + regr_count(Y,X) Number of non-null input rows. + regr_intercept(Y,X) Y-intercept of the least-squares-fit linear equation determined by the (X,Y) pairs. + regr_r2(Y,X) Square of the correlation coefficient. + regr_slope(Y,X) Slope of the least-squares-fit linear equation determined by the (X,Y) pairs. + regr_sxx(Y,X) Sum of squares of the independent variable. + regr_sxy(Y,X) Sum of products of independent times dependent variable. + regr_syy(Y,X) Sum of squares of the dependent variable. + stddev_samp(expression) Sample standard deviation of the input values. + var_samp(expression) Sample variance of the input values (square of the sample standard deviation). + SQL Functions - General + + Many of these functions are similar to standard SQL functions -- see the JBDC escape syntax documentation. + Function Description + abs(value) Returns the absolute value. + acos(value) Returns the arc cosine. + age(date1, date2) + + Supplies the difference in age between the two dates, calculated in years. + age(date1, date2, interval) + + The interval indicates the unit of age measurement, either SQL_TSI_MONTH or SQL_TSI_YEAR. + age_in_months(date1, date2) Behavior is undefined if date2 is before date1. + age_in_years(date1, date2) Behavior is undefined if date2 is before date1. + asin(value) Returns the arc sine. + atan(value) Returns the arc tangent. + atan2(value1, value2) Returns the arctangent of the quotient of two values. + case + + CASE can be used to test various conditions and return various results based on the test. You can use either simple CASE or searched CASE syntax. In the following examples "value#" indicates a value to match against, where "test#" indicates a boolean expression to evaluate. In the "searched" syntax, the first test expression that evaluates to true will determine which result is returned. Note that the LabKey SQL parser sometimes requires the use of additional parentheses within the statement. + + CASE (value) WHEN (value1) THEN (result1) ELSE (result2) END + CASE (value) WHEN (value1) THEN (result1) WHEN (value2) THEN (result2) ELSE (resultDefault) END + CASE WHEN (test1) THEN (result1) ELSE (result2) END + CASE WHEN (test1) THEN (result1) WHEN (test2) THEN (result2) WHEN (test3) THEN (result3) ELSE (resultDefault) END + + Example: + + SELECT "StudentName", + School, + CASE WHEN (Division = 'Grades 3-5') THEN (Scores.Score*1.13) ELSE Score END AS AdjustedScore, + Division + FROM Scores + ceiling(value) Rounds the value up. + coalesce(value1,...,valueN) Returns the first non-null value in the argument list. Use to set default values for display. + concat(value1,value2) Concatenates two values. + contextPath() Returns the context path starting with “/” (e.g. “/labkey”). Returns the empty string if there is no current context path. (Returns VARCHAR.) + cos(radians) Returns the cosine. + cot(radians) Returns the cotangent. + curdate() Returns the current date. + curtime() Returns the current time + dayofmonth(date) Returns the day of the month (1-31) for a given date. + dayofweek(date) Returns the day of the week (1-7) for a given date. (Sun=1 and Sat=7) + dayofyear(date) Returns the day of the year (1-365) for a given date. + degrees(radians) Returns degrees based on the given radians. + exp(n) Returns Euler's number e raised to the nth power. e = 2.71828183 + floor(value) Rounds down to the nearest integer. + folderName() LabKey SQL extension function. Returns the name of the current folder, without beginning or trailing "/". (Returns VARCHAR.) + folderPath() LabKey SQL extension function. Returns the current folder path (starts with “/”, but does not end with “/”). The root returns “/”. (Returns VARCHAR.) + greatest(a, b, c, ...) Returns the greatest value from the list expressions provided. Any number of expressions may be used. The expressions must have the same data type, which will also be the type of the result. The LEAST() function is similar, but returns the smallest value from the list of expressions. GREATEST() and LEAST() are not implemented for SAS databases. + + When NULL values appear in the list of expressions, different database implementations as follows: + + - PostgreSQL & MS SQL Server ignore NULL values in the arguments, only returning NULL if all arguments are NULL. + - Oracle and MySql return NULL if any one of the arguments are NULL. Best practice: wrap any potentially nullable arguments in coalesce() or ifnull() and determine at the time of usage if NULL should be treated as high or low. + + Example: + + SELECT greatest(score_1, score_2, score_3) As HIGH_SCORE + FROM MyAssay + hour(time) Returns the hour for a given date/time. + ifdefined(column_name) IFDEFINED(NAME) allows queries to reference columns that may not be present on a table. Without using IFDEFINED(), LabKey will raise a SQL parse error if the column cannot be resolved. Using IFDEFINED(), a column that cannot be resolved is treated as a NULL value. The IFDEFINED() syntax is useful for writing queries over PIVOT queries or assay tables where columns may be added or removed by an administrator. + ifnull(testValue, defaultValue) If testValue is null, returns the defaultValue. Example: IFNULL(Units,0) + isequal LabKey SQL extension function. ISEQUAL(a,b) is equivalent to (a=b OR (a IS NULL AND b IS NULL)) + ismemberof(groupid) LabKey SQL extension function. Returns true if the current user is a member of the specified group. + javaConstant(fieldName) LabKey SQL extension function. Provides access to public static final variable values. For details see LabKey SQL Utility Functions. + lcase(string) Convert all characters of a string to lower case. + least(a, b, c, ...) Returns the smallest value from the list expressions provided. For more details, see greatest() above. + left(string, integer) Returns the left side of the string, to the given number of characters. Example: SELECT LEFT('STRINGVALUE',3) returns 'STR' + length(string) Returns the length of the given string. + locate(substring, string) locate(substring, string, startIndex) Returns the location of the first occurrence of substring within string. startIndex provides a starting position to begin the search. + log(n) Returns the natural logarithm of n. + log10(n) Base base 10 logarithm on n. + lower(string) Convert all characters of a string to lower case. + ltrim(string) Trims white space characters from the left side of the string. For example: LTRIM(' Trim String') + minute(time) Returns the minute value for the given time. + mod(dividend, divider) Returns the remainder of the division of dividend by divider. + moduleProperty(module name, property name) + + LabKey SQL extension function. Returns a module property, based on the module and property names. For details see LabKey SQL Utility Functions. + month(date) Returns the month value (1-12) of the given date. + monthname(date) Return the month name of the given date. + now() Returns the system date and time. + overlaps LabKey SQL extension function. Supported only when PostgreSQL is installed as the primary database. \s + + SELECT OVERLAPS (START1, END1, START2, END2) AS COLUMN1 FROM MYTABLE + + The LabKey SQL syntax above is translated into the following PostgreSQL syntax: \s + + SELECT (START1, END1) OVERLAPS (START2, END2) AS COLUMN1 FROM MYTABLE + pi() Returns the value of pi;. + power(base, exponent) Returns the base raised to the power of the exponent. For example, power(10,2) returns 100. + quarter(date) Returns the yearly quarter for the given date where the 1st quarter = Jan 1-Mar 31, 2nd quarter = Apr 1-Jun 30, 3rd quarter = Jul 1-Sep 30, 4th quarter = Oct 1-Dec 31 + radians(degrees) Returns the radians for the given degrees. + rand(), rand(seed) Returns a random number between 0 and 1. + repeat(string, count) Returns the string repeated the given number of times. SELECT REPEAT('Hello',2) returns 'HelloHello'. + round(value, precision) Rounds the value to the specified number of decimal places. ROUND(43.3432,2) returns 43.34 + rtrim(string) Trims white space characters from the right side of the string. For example: RTRIM('Trim String ') + second(time) Returns the second value for the given time. + sign(value) Returns the sign, positive or negative, for the given value. + sin(value) Returns the sine for the given value. + startswith(string, prefix) Tests to see if the string starts with the specified prefix. For example, STARTSWITH('12345','2') returns FALSE. + sqrt(value) Returns the square root of the value. + substring(string, start, length) Returns a portion of the string as specified by the start location (1-based) and length (number of characters). For example, substring('SomeString', 1,2) returns the string 'So'. + tan(value) + + Returns the tangent of the value. + timestampadd(interval, number_to_add, timestamp) + + Adds an interval to the given timestamp value. The interval value must be surrounded by quotes. Possible values for interval: + + SQL_TSI_FRAC_SECOND + SQL_TSI_SECOND + SQL_TSI_MINUTE + SQL_TSI_HOUR + SQL_TSI_DAY + SQL_TSI_WEEK + SQL_TSI_MONTH + SQL_TSI_QUARTER + SQL_TSI_YEAR + + Example: TIMESTAMPADD('SQL_TSI_QUARTER', 1, "Physical Exam".date) AS NextExam + timestampdiff(interval, timestamp1, timestamp2) + + Finds the difference between two timestamp values at a specified interval. The interval must be surrounded by quotes. + + Example: TIMESTAMPDIFF('SQL_TSI_DAY', SpecimenEvent.StorageDate, SpecimenEvent.ShipDate) + + Note that PostgreSQL does not support the following intervals: + + SQL_TSI_FRAC_SECOND + SQL_TSI_WEEK + SQL_TSI_MONTH + SQL_TSI_QUARTER + SQL_TSI_YEAR + + As a workaround, use the 'age' functions defined above. + truncate(numeric value, precision) Truncates the numeric value to the precision specified. This is an arithmetic truncation, not a string truncation. + TRUNCATE(123.4567,1) returns 123.4 + TRUNCATE(123.4567,2) returns 123.45 + TRUNCATE(123.4567,-1) returns 120.0 + + May require an explict CAST into NUMERIC, as LabKey SQL does not check data types for function arguments. + + SELECT + PhysicalExam.Temperature, + TRUNCATE(CAST(Temperature AS NUMERIC),1) as truncTemperature + FROM PhysicalExam + ucase(string), upper(string) Converts all characters to upper case. + userid() LabKey SQL extension function. Returns the userid, an integer, of the logged in user. + username() LabKey SQL extension function. Returns the current user display name. VARCHAR + version() LabKey SQL extension function. Returns the current schema version of the core module as a NUMERIC with four decimal places. For example: 20.0070 + week(date) Returns the week value (1-52) of the given date. + year(date) Return the year of the given date. Assuming the system date is March 4 2023, then YEAR(NOW()) return 2023. + SQL Functions - PostgreSQL Specific + + LabKey SQL supports the following PostgreSQL functions. + See the PostgreSQL docs for usage details. + PostgreSQL Function Docs\s + ascii(value) Returns the ASCII code of the first character of value. \s + btrim(value, + trimchars) Removes characters in trimchars from the start and end of string. trimchars defaults to white space. + + BTRIM(' trim ') returns TRIM\s + BTRIM('abbatrimabtrimabba', 'ab') returns trimabtrim + + character_length(value), char_length(value) + Returns the number of characters in value. + chr(integer_code) Returns the character with the given integer_code. + + CHR(70) returns F + concat_ws(sep text, + val1 "any" [, val2 "any" [,...]]) -> text Concatenates all but the first argument, with separators. The first argument is used as the separator string, and should not be NULL. Other NULL arguments are ignored. See the PostgreSQL docs. + + concat_ws(',', 'abcde', 2, NULL, 22) → abcde,2,22 + decode(text, + format) See the PostgreSQL docs. + encode(binary, + format) See the PostgreSQL docs. + is_distinct_from(a, b) OR + is_not_distinct_from(a, b) Not equal (or equal), treating null like an ordinary value. + initcap(string) Converts the first character of each separate word in string to uppercase and the rest to lowercase. + lpad(string,\s + int, + fillchars) Pads string to length int by prepending characters fillchars. + md5(text) Returns the hex MD5 value of text. + octet_length(string) Returns the number of bytes in string. + overlaps See above for syntax details. + quote_ident(string) Returns string quoted for use as an identifier in an SQL statement. + quote_literal(string) Returns string quoted for use as a string literal in an SQL statement. + regexp_replace See PostgreSQL docs for details: reference doc, example doc + repeat(string, int) Repeats string the specified number of times. + replace(string,\s + matchString,\s + replaceString) Searches string for matchString and replaces occurrences with replaceString. + rpad(string,\s + int, + fillchars) Pads string to length int by postpending characters fillchars. + similar_to(A,B,C) String pattern matching using SQL regular expressions. 'A' similar to 'B' escape 'C'. See the PostgreSQL docs. + split_part(string, + delimiter, + int) Splits string on delimiter and returns fragment number int (starting the count from 1). + + SPLIT_PART('mississippi', 'i', 4) returns 'pp'. + string_to_array See Array Functions in the PostgreSQL docs. + strpos(string, + substring) Returns the position of substring in string. (Count starts from 1.) + substr(string, + fromPosition, + charCount) + + Extracts the number of characters specified by charCount from string starting at position fromPosition. + + SUBSTR('char_sequence', 5, 2) returns '_s'\s + to_ascii(string, + encoding) Convert string to ASCII from another encoding. + to_hex(int) Converts int to its hex representation. + translate(text, + fromText, + toText) Characters in string matching a character in the fromString set are replaced by the corresponding character in toString. + to_char See Data Type Formatting Functions in the PostgreSQL docs. + to_date(textdate, + format) See Data Type Formatting Functions in the PostgreSQL docs.\s + to_timestamp See Data Type Formatting Functions in the PostgreSQL docs. + to_number See Data Type Formatting Functions in the PostgreSQL docs. + unnest See Array Functions in the PostgreSQL docs. + JSON and JSONB Operators and Functions + + LabKey SQL supports the following PostgreSQL JSON and JSONB operators and functions. Note that LabKey SQL does not natively understand arrays and some other features, but it may still be possible to use the functions that expect them. + See the PostgreSQL docs for usage details. + PostgreSQL Operators and Functions Docs + ->, ->>, #>, #>>, @>, <@, ?, ?|, ?&, ||, -, #- + LabKey SQL supports these operators via a pass-through function, json_op. The function's first argument is the operator's first operand. The first second is the operator, passed as a string constant. The function's third argument is the second operand. For example, this Postgres SQL expression: + + a_jsonb_column --> 2 + + can be represented in LabKey SQL as: + + json_op(a_jsonb_column, '-->', 2) + + parse_json, parse_jsonb Casts a text value to a parsed JSON or JSONB data type. For example, + + '{"a":1, "b":null}'::jsonb + + or + + CAST('{"a":1, "b":null}' AS JSONB) + + can be represented in LabKey SQL as: + + parse_jsonb('{"a":1, "b":null}') + + to_json, to_jsonb Converts a value to the JSON or JSONB data type. Will treat a text value as a single JSON string value + array_to_json Converts an array value to the JSON data type. + row_to_json Converts a scalar (simple value) row to JSON. Note that LabKey SQL does not support the version of this function that will convert an entire table to JSON. Consider using "to_jsonb()" instead. + json_build_array, jsonb_build_array Build a JSON array from the arguments + json_build_object, jsonb_build_object Build a JSON object from the arguments + json_object, jsonb_object Build a JSON object from a text array + json_array_length, jsonb_array_length Return the length of the outermost JSON array + json_each, jsonb_each Expand the outermost JSON object into key/value pairs. Note that LabKey SQL does not support the table version of this function. Usage as a scalar function like this is supported: + + SELECT json_each('{"a":"foo", "b":"bar"}') AS Value + + json_each_text, jsonb_each_text Expand the outermost JSON object into key/value pairs into text. Note that LabKey SQL does not support the table version of this function. Usage as a scalar function (similar to json_each) is supported. + json_extract_path, jsonb_extract_path Return the JSON value referenced by the path + json_extract_path_text, jsonb_extract_path_text Return the JSON value referenced by the path as text + json_object_keys, jsonb_object_keys Return the keys of the outermost JSON object + json_array_elements, jsonb_array_elements Expand a JSON array into a set of values + json_array_elements_text, jsonb_array_elements_text Expand a JSON array into a set of text values + json_typeof, jsonb_typeof Return the type of the outermost JSON object + json_strip_nulls, jsonb_strip_nulls Remove all null values from a JSON object + jsonb_insert Insert a value within a JSON object at a given path + jsonb_pretty Format a JSON object as indented text + jsonb_set Set the value within a JSON object for a given path. Strict, i.e. returns NULL on NULL input. + jsonb_set_lax Set the value within a JSON object for a given path. Not strict; expects third argument to specify how to treat NULL input (one of 'raise_exception', 'use_json_null', 'delete_key', or 'return_target'). + jsonb_path_exists, jsonb_path_exists_tz Checks whether the JSON path returns any item for the specified JSON value. The "_tz" variant is timezone aware. + jsonb_path_match, jsonb_path_match_tz Returns the result of a JSON path predicate check for the specified JSON value. The "_tz" variant is timezone aware. + jsonb_path_query, jsonb_path_query_tz Returns all JSON items returned by the JSON path for the specified JSON value. The "_tz" variant is timezone aware. + jsonb_path_query_array, jsonb_path_query_array_tz Returns as an array, all JSON items returned by the JSON path for the specified JSON value. The "_tz" variant is timezone aware. + jsonb_path_query_first, jsonb_path_query_first_tz Returns the first JSON item returned by the JSON path for the specified JSON value. The "_tz" variant is timezone aware. + General Syntax + Syntax Item Description + Case Sensitivity Schema names, table names, column names, SQL keywords, function names are case-insensitive in LabKey SQL. + Comments Comments that use the standard SQL syntax can be included in queries. '--' starts a line comment. Also, '/* */' can surround a comment block: + + -- line comment 1 + -- line comment 2 + /* block comment 1 + block comment 2 */ + SELECT ...\s + Identifiers Identifiers in LabKey SQL may be quoted using double quotes. (Double quotes within an identifier are escaped with a second double quote.) + + SELECT "Physical Exam".* + ...\s + Lookups Lookups columns reference data in other tables. In SQL terms, they are foreign key columns. See Lookups for details on creating lookup columns. Lookups use a convenient syntax of the form "Table.ForeignKey.FieldFromForeignTable" to achieve what would normally require a JOIN in SQL. Example: + + Issues.AssignedTo.DisplayName + String Literals String literals are quoted with single quotes ('). Within a single quoted string, a single quote is escaped with another single quote. + + SELECT * FROM TableName WHERE FieldName = 'Jim''s Item'\s + Date/Time Literals + + Date and Timestamp (Date&Time) literals can be specified using the JDBC escape syntax + + {ts '2001-02-03 04:05:06'} + + {d '2001-02-03'}\s + """; + } + Content contentFromText(String s) + { + return Content.fromParts(Part.fromText(s)); + } + + String getModel() + { + return "gemini-2.5-flash"; + } + + Chat getChat(String currentSchema) + { + return SessionHelper.getAttribute(getViewContext().getRequest(), Chat.class.getName(), () -> { + Client client = new Client(); + + Schema columnsMetaDataParameters = Schema.builder() + .type(Type.Known.OBJECT) + .properties(Map.of("quotedTableName", Schema.builder() + .type(Type.Known.STRING) + .description("Fully qualified table name as it would appear in SQL e.g. \"schema\".\"table\"") + .build())) + .required("quotedTableName") + .build(); + var listColumnMetaDataFn = FunctionDeclaration.builder().name("listColumnsForTable").description("Provide column metadata for a sql table.").parameters(columnsMetaDataParameters); + + Schema tablesMetaDataParameters = Schema.builder() + .type(Type.Known.OBJECT) + .properties(Map.of("quotedSchemaName", Schema.builder() + .type(Type.Known.STRING) + .description("Fully qualified schema name as it would appear in SQL e.g. \"schema\"") + .build())) + .required("quotedSchemaName") + .build(); + var listTablesMetaDataFn = FunctionDeclaration.builder().name("listTablesForSchema").description("Provide column metadata for a sql table.").parameters(tablesMetaDataParameters); + + var tools = Tool.builder().functionDeclarations(listColumnMetaDataFn, listTablesMetaDataFn); + + GenerateContentConfig config = GenerateContentConfig.builder().tools(tools).build(); + + Chat chatSession = client.chats.create(getModel(), config); + + StringBuilder serviceMessage = new StringBuilder(); + serviceMessage.append("Your job is to generate SQL statements. Here is some reference material you can use:\n\n").append(getSQLHelp()); + + QuerySchema defaultSchema = DefaultSchema.get(getUser(), getContainer()); + StringBuilder sb = new StringBuilder(); + for (var name : defaultSchema.getSchemaNames()) + { + var schema = defaultSchema.getSchema(name); + if (null != schema) + { + sb.append("\t* ").append(schema.getSchemaPath().toSQLString()); + if (isNotBlank(schema.getDescription())) + sb.append("\t").append(schema.getDescription()); + sb.append("\n"); + } + } + serviceMessage.append("\n\nHere are the available schemas:\n" + sb); + + if (!isBlank(currentSchema)) + { + var schema = defaultSchema.getSchema(currentSchema); + if (null != schema) + { + serviceMessage.append("\n\nCurrent default schema is " + schema.getSchemaPath().toSQLString() + ". This is a list of tables in this schema formatted as JSON\n```" + + listColumnsForTable(schema.getSchemaPath().toSQLString()) + "\n```"); + } + } + + chatSession.sendMessage(serviceMessage.toString()); + return chatSession; + }); + } + + @Override + public Object execute(PromptForm form, BindException errors) throws Exception + { + Chat chatSession = getChat(form.getSchemaName()); + + String prompt = form.getPrompt(); + for (int retry=0 ; retry < 5 ; retry++) + { + GenerateContentResponse response = chatSession.sendMessage(prompt); + var functionCalls = response.functionCalls(); + if (null == functionCalls || functionCalls.isEmpty()) + { + var sql = extractSql(response.text()); + var text = null==sql ? response.text() : null; + + /* VALIDATE SQL */ + if (null != sql) + { + QuerySchema schema = DefaultSchema.get(getUser(), getContainer()).getSchema("study"); + try + { + TableInfo ti = QueryService.get().createTable(schema, sql, null, true); + } + catch (QueryException x) + { + String validationPrompt = "That SQL caused the error below, can you attempt to fix this?\n```" + x.getMessage() + "```"; + response = chatSession.sendMessage(validationPrompt); + text = response.text(); + var newSQL = extractSql(response.text()); + if (isNotBlank(newSQL)) + sql = newSQL; + } + } + + System.err.println(chatSession.getHistory(true)); + var ret = new JSONObject(Map.of( + "model", getModel(), + "success", Boolean.TRUE)); + if (null != sql) + ret.put("sql",sql); + if (null != text) + ret.put("text", text); + return ret; + } + StringBuilder fnPrompt = new StringBuilder(); + for (var functionCall : response.functionCalls()) + { + var functionName = functionCall.name().orElse(null); + if ("listColumnsForTable".equals(functionName)) + { + var quotedName = String.valueOf(functionCall.args().get().get("quotedTableName")); + var res = listColumnsForTable(quotedName); + fnPrompt.append("Here is additional metadata for table " + quotedName + " formatted as JSON:\n```").append(res).append("```\n\n"); + } + else if ("listTablesForSchema".equals(functionName)) + { + var quotedName = String.valueOf(functionCall.args().get().get("quotedSchemaName")); + var res = listTablesForSchema(quotedName); + fnPrompt.append("Here is additional metadata for schema " + quotedName + " formatted as JSON:\n```").append(res).append("```\n\n"); + } + } + prompt = fnPrompt.toString(); + } + + // FALLBACK??? + GenerateContentResponse response = chatSession.sendMessage(form.getPrompt()); + System.err.println(chatSession.getHistory(true)); + var ret = new JSONObject(Map.of( + "text", response.text(), + "user", getViewContext().getUser().getName(), + "model", getModel(), + "success", Boolean.TRUE)); + var functionCalls = response.functionCalls(); + if (null != functionCalls && !functionCalls.isEmpty()) + ret.put("functionCall", functionCalls.get(0).toString()); + return ret; + } + + String extractSql(String text) + { + if (text.startsWith("SELECT ")) + return text; + if (text.startsWith("WITH ") && text.contains("SELECT ")) + return text; + if (text.startsWith("PARAMETERS ") && text.contains("SELECT ")) + return text; + var sql = text.indexOf("```sql\n"); + if (sql >= 0) + { + var end = text.indexOf("```", sql+7); + if (end >= 0) + return text.substring(7,end); + } + return null; + } + } + + + @RequiresPermission(ReadPermission.class) + @RequiresLogin + public static class ResetQueryAgentAction extends ReadOnlyApiAction + { + @Override + public Object execute(Object o, BindException errors) throws Exception + { + var session = getViewContext().getRequest().getSession(false); + if (null != session) + session.removeAttribute(Chat.class.getName()); + return new JSONObject(Map.of("success", Boolean.TRUE)); + } + } + + public static JSONObject listTablesForSchema(String fullQuotedName) + { + SchemaKey fullKey; + + // TODO : correct method for parsing quoted identifier + if (fullQuotedName.startsWith("\"") && fullQuotedName.endsWith("\"")) + { + String[] parts = StringUtils.strip(fullQuotedName, "\"").split("\"\\.\""); + fullKey = SchemaKey.fromParts(parts); + } + else + { + String[] parts = StringUtils.split(fullQuotedName, "."); + fullKey = SchemaKey.fromParts(parts); + } + + ViewContext context = HttpView.currentView().getViewContext(); + var defaultSchema = DefaultSchema.get(context.getUser(), context.getContainer()); + var schema = DefaultSchema.resolve(defaultSchema, fullKey); + if (null == schema) + return new JSONObject("error", "could not find schema for : " + fullQuotedName); + + JSONArray array = new JSONArray(); + for (String tableName : schema.getTableNames()) + { + // CONSIDER schema.getTableDescription()??? + TableInfo td = schema.getTable(tableName, null); + if (null == td) + continue; + JSONObject table = new JSONObject(); + table.put("schemaName", schema.getName()); + table.put("tableName", td.getName()); + table.put("fullQuotedName", new SchemaKey(schema.getSchemaPath(), td.getName()).toSQLString()); + table.put("description", td.getDescription()); + array.put(table); + } + return new JSONObject(Map.of("tables", array)); + } + + public static JSONObject listColumnsForTable(String fullQuotedName) + { + SchemaKey fullKey; + + // TODO : correct method for parsing quoted identifier + if (fullQuotedName.startsWith("\"") && fullQuotedName.endsWith("\"")) + { + String[] parts = StringUtils.strip(fullQuotedName, "\"").split("\"\\.\""); + fullKey = SchemaKey.fromParts(parts); + } + else + { + String[] parts = StringUtils.split(fullQuotedName, "."); + fullKey = SchemaKey.fromParts(parts); + } + + SchemaKey schemaKey; + String tableName; + if (fullKey.size() > 1) + { + schemaKey = fullKey.getParent(); + tableName = fullKey.getName(); + } + else if (fullKey.size() == 1) + { + schemaKey = SchemaKey.fromParts("study"); + tableName = fullKey.getName(); + } + else + { + return new JSONObject("error", "could not find table"); + } + + ViewContext context = HttpView.currentView().getViewContext(); + var defaultSchema = DefaultSchema.get(context.getUser(), context.getContainer()); + + JSONArray array = new JSONArray(); + var schema = DefaultSchema.resolve(defaultSchema, schemaKey); + if (null == schema) + return new JSONObject("error", "could not find table"); + + TableInfo td = schema.getTable(tableName, null); + if (null == td) + return new JSONObject("error", "could not find table"); + + JSONObject table = new JSONObject(); + table.put("schemaName", schema.getName()); + table.put("tableName", td.getName()); + table.put("fullQuotedName", new SchemaKey(schema.getSchemaPath(), td.getName()).toSQLString()); + table.put("description", td.getDescription()); + + JSONArray columns = new JSONArray(); + for (ColumnInfo col : td.getColumns()) + { + JSONObject md = new JSONObject(); + md.put("name", col.getName()); + md.put("label", col.getLabel()); + md.put("type", col.getJdbcType().name()); + md.put("description", col.getDescription()); + columns.put(md); + // TODO PK/FK + } + table.put("columns",columns); + + return table; + } + + + /** JSON schema example provided by GEMINI, using triple tick-marks to delimit the machine-readable structured data + * + * Here is the database schema in JSON format: + * ```{ + * "database": "ecommerce", + * "tables": [ + * { + * "name": "customers", + * "description": "Stores customer details, including their contact information and location.", + * "columns": [ + * {"name": "customer_id", "type": "INTEGER", "description": "Unique identifier for each customer.", "is_primary_key": true}, + * {"name": "first_name", "type": "VARCHAR(50)", "description": "The customer's first name."}, + * {"name": "email", "type": "VARCHAR(100)", "description": "The customer's email address.", "is_unique": true} + * ] + * }, + * { + * "name": "orders", + * "description": "Tracks customer orders.", + * "columns": [ + * {"name": "order_id", "type": "INTEGER", "description": "Unique identifier for each order.", "is_primary_key": true}, + * {"name": "customer_id", "type": "INTEGER", "description": "Foreign key linking to the customers table.", "is_foreign_key": true, "references": "customers.customer_id"}, + * {"name": "order_date", "type": "DATE", "description": "The date the order was placed."} + * ] + * } + * ] + * }``` + */ } diff --git a/query/src/org/labkey/query/view/sourceQuery.jsp b/query/src/org/labkey/query/view/sourceQuery.jsp index 0c87691e053..2ff8af896ac 100644 --- a/query/src/org/labkey/query/view/sourceQuery.jsp +++ b/query/src/org/labkey/query/view/sourceQuery.jsp @@ -83,133 +83,318 @@ border: none; } + /* chat history */ + DIV.chatItem { + /* width: 200px; */ + background-color: #4CAF50; + border-radius: 15px; + display: flex; + /* justify-content: center; + align-items: center; */ + box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1); /* Optional shadow effect */ + } + + DIV.userPrompt { + margin: 5px; + margin-right: 20px; + background-color : white; + } + + DIV.genaiResponse { + margin: 5px; + margin-left: 20px; + background-color: lightgray; + } + + DIV.sqlResponse { + margin-left: 10px; + background-color: pink; + } + - -
+ +<%-- should use Ext4 Panel for layout, but this is just a prototype anyway --%> + + + +
+ +
+ 		+
+ +
+ + diff --git a/query/webapp/query/QueryEditorPanel.js b/query/webapp/query/QueryEditorPanel.js index 319c2a48587..2f6f2a4ea65 100644 --- a/query/webapp/query/QueryEditorPanel.js +++ b/query/webapp/query/QueryEditorPanel.js @@ -304,6 +304,15 @@ Ext4.define('LABKEY.query.SourceEditorPanel', { if (this.codeMirror) queryText = this.codeMirror.getValue(); return queryText; + }, + + setValue : function(queryText) + { + this.query.queryText = queryText; + if (this.codeMirror) + { + return this.codeMirror.setValue(queryText); + } } }); From deb64b1d82fa5cdcb01cc1fe1e9090263d05564d Mon Sep 17 00:00:00 2001 From: labkey-matthewb Date: Mon, 15 Sep 2025 11:57:37 -0700 Subject: [PATCH 02/25] pk/fk --- .../query/controllers/QueryController.java | 31 +++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/query/src/org/labkey/query/controllers/QueryController.java b/query/src/org/labkey/query/controllers/QueryController.java index 83340488f35..ee424e00487 100644 --- a/query/src/org/labkey/query/controllers/QueryController.java +++ b/query/src/org/labkey/query/controllers/QueryController.java @@ -179,6 +179,7 @@ import org.labkey.api.query.QueryAction; import org.labkey.api.query.QueryDefinition; import org.labkey.api.query.QueryException; +import org.labkey.api.query.QueryForeignKey; import org.labkey.api.query.QueryForm; import org.labkey.api.query.QueryParam; import org.labkey.api.query.QueryParseException; @@ -9677,7 +9678,7 @@ String extractSql(String text) { var end = text.indexOf("```", sql+7); if (end >= 0) - return text.substring(7,end); + return text.substring(sql+7,end); } return null; } @@ -9788,6 +9789,8 @@ else if (fullKey.size() == 1) table.put("fullQuotedName", new SchemaKey(schema.getSchemaPath(), td.getName()).toSQLString()); table.put("description", td.getDescription()); + var pkColumns = td.getPkColumns(); + var pk = pkColumns.size() == 1 ? pkColumns.get(0).getFieldKey() : null; JSONArray columns = new JSONArray(); for (ColumnInfo col : td.getColumns()) { @@ -9796,8 +9799,32 @@ else if (fullKey.size() == 1) md.put("label", col.getLabel()); md.put("type", col.getJdbcType().name()); md.put("description", col.getDescription()); + if (col.getFieldKey().equals(pk)) + md.put("is_primary_key", Boolean.TRUE); + var fk = col.getFk(); + if (null != fk) + { + if (fk instanceof QueryForeignKey qfk) + { + SchemaKey qfkSchema = qfk.getLookupSchemaKey(); + SchemaKey qfkTable = new SchemaKey(qfkSchema, qfk.getLookupTableName()); + SchemaKey qfkColumn = new SchemaKey(qfkTable, qfk.getLookupColumnName()); + md.put("is_foreign_key", Boolean.TRUE); + md.put("references", qfkColumn.toSQLString()); + } + else + { + TableDescription references = fk.getLookupTableDescription(); + if (null != references && references.isPublic()) + { + SchemaKey qfkTable = SchemaKey.fromParts(td.getSchema().getQuerySchemaName(), td.getName()); + SchemaKey qfkColumn = new SchemaKey(qfkTable, fk.getLookupColumnName()); + md.put("is_foreign_key", Boolean.TRUE); + md.put("references", qfkColumn.toSQLString()); + } + } + } columns.put(md); - // TODO PK/FK } table.put("columns",columns); From ac073788abfdff8c70b94882a8b69f58ee2a428c Mon Sep 17 00:00:00 2001 From: labkey-matthewb Date: Mon, 15 Sep 2025 13:36:54 -0700 Subject: [PATCH 03/25] LabKeySql.md --- .../org/labkey/query/controllers/LabKeySql.md | 212 ++++++ .../query/controllers/QueryController.java | 676 +----------------- 2 files changed, 233 insertions(+), 655 deletions(-) create mode 100644 query/src/org/labkey/query/controllers/LabKeySql.md diff --git a/query/src/org/labkey/query/controllers/LabKeySql.md b/query/src/org/labkey/query/controllers/LabKeySql.md new file mode 100644 index 00000000000..9beef68e462 --- /dev/null +++ b/query/src/org/labkey/query/controllers/LabKeySql.md @@ -0,0 +1,212 @@ +### **LabKey SQL Documentation** + +LabKey SQL is a unique SQL dialect that extends standard SQL functionality with features tailored for the LabKey Server platform, particularly for scientific data management. + +----- + +### **1. Lookups and Joins** + +LabKey SQL simplifies data joining by providing an intuitive **lookup syntax** that often eliminates the need for explicit `JOIN` statements. + +* **Syntax:** + `SELECT parent_table.lookup_column.target_column FROM parent_table` +* **Functionality:** + This syntax allows you to access columns from a foreign table by following the lookup relationship with dot notation. This is a powerful feature for simplifying queries that involve related data. +* **Examples:** + * **Simple Lookup:** To retrieve a participant's gender from the `Demographics` table using a lookup from the `PhysicalExam` table: + ```sql + SELECT PhysicalExam.ParticipantId, PhysicalExam.weight_kg, Demographics.Gender, Demographics.Height + FROM PhysicalExam + ``` + * **Joining Across Folders:** To join data from a `Demographics` table in one folder with a `Languages` list in a different folder: + ```sql + JOIN "/Other/Folder".lists.Languages ON Demographics.Language=Languages.Language + ``` + +----- + +### **2. Calculated Columns** + +LabKey SQL allows you to create virtual columns within a query by using SQL expressions. These calculated columns are not stored in the database but are computed on the fly. + +* **Syntax:** + `SELECT expression AS column_name FROM table` +* **Functionality:** + The syntax involves performing a calculation and then aliasing the result with a new column name using the `as` keyword. +* **Examples:** + * **Pulse Pressure:** To calculate pulse pressure from systolic and diastolic blood pressure values: + ```sql + PhysicalExam.systolicBP-PhysicalExam.diastolicBP as PulsePressure + ``` + * **BMI (Body Mass Index):** A more complex example that uses an intermediate query to calculate BMI from height and weight data: + ```sql + ROUND(weight_kg / (height_m * height_m), 2) AS BMI + ``` + +----- + +### **3. Pivoting** + +A `PIVOT` query helps you summarize and re-visualize data by transforming rows into columns. + +* **Syntax for PIVOT...BY Query:** + A pivot query is a `SELECT` statement specifying how to pivot and group columns. The basic syntax is `PIVOT [aggregating_column] BY [pivoting_column]`. +* **PIVOT...BY...IN Syntax:** + You can use an `IN` clause to specify a fixed set of column names to pivot. This is more efficient. + ```sql + PIVOT new_column_name BY pivoting_column IN ('value1', 'value2') + ``` + Note that pivot column names are case-sensitive. You may need to use `LOWER()` or `UPPER()` in your query to work around this issue. +* **Pivoting by Two Columns:** + Two levels of `PIVOT` are not directly supported. However, you can achieve a similar result by concatenating the two values together and pivoting on that "calculated" column. + ```sql + SELECT + Run.SampleCondition || ' ' || PeakLabel AS ConditionPeak, + AVG(Data.PercTimeCorrArea) AS AvgPercTimeCorrArea + FROM Data + GROUP BY Run.SampleCondition || ' ' || PeakLabel + PIVOT AvgPercTimeCorrArea BY ConditionPeak + ``` + +----- + +### **4. Cross-Folder Queries** + +You can write queries that access data from different folders within the LabKey Server instance, allowing for data integration across projects. + +* **Syntax:** + `SELECT * FROM Project."folder_path".schema.table` +* **Functionality:** + The folder path is a dot-delimited string that specifies the location of the table, including the project name. The user must have "Reader" permissions in each folder referenced in the query. +* **Example:** + ```sql + SELECT + p.ParticipantID, + ROUND(AVG(p.Temp_C), 1) AS AverageTemp + FROM Project."Tutorials/Demo/".study."Physical Exam" p + GROUP BY p.ParticipantID + ``` + +----- + +### **5. Parameterized Queries** + +LabKey SQL supports **parameterized queries** to improve security and reusability. + +* **Syntax:** + `PARAMETERS(param1, param2) SELECT * FROM table WHERE column = param1` +* **Functionality:** + The `PARAMETERS` keyword declares parameters that can be passed into the query. +* **Example:** A query with two parameters, `MinTemp` and `MinWeight`: + ```sql + PARAMETERS(MinTemp double, MinWeight double) + SELECT + ParticipantID, + temperature_C, + weight_kg + FROM PhysicalExam + WHERE temperature_C >= MinTemp AND weight_kg >= MinWeight + ``` + +----- + +### **6. Metadata Annotations** + +LabKey SQL allows you to directly annotate your SQL statements to override how column metadata is displayed in the LabKey interface. + +* **Syntax:** + `SELECT column_name @annotation FROM table` +* **Functionality:** + Annotations control the display of a column without changing the underlying data. +* **Examples:** + * **Hiding a Column:** + ```sql + SELECT ratio @hidden, log(ratio) as log_ratio + ``` + * **Setting a Title and Format:** + ```sql + SELECT 10/7.0 AS Num @title='Calculated Number' @Format='0.00' + ``` + +----- + +### **7. Available Methods** + +Here is a summary of the available functions and methods in LabKey SQL. + +#### **Mathematical Functions** + +* `abs(value)`: Returns the absolute value. +* `acos(value)`: Returns the arc cosine. +* `asin(value)`: Returns the arc sine. +* `atan(value)`: Returns the arc tangent. +* `atan2(value1, value2)`: Returns the arctangent of the quotient. +* `ceiling(value)`: Rounds the value up. +* `cos(radians)`: Returns the cosine. +* `cot(radians)`: Returns the cotangent. +* `degrees(radians)`: Returns degrees. +* `exp(n)`: Returns Euler's number 'e' raised to the nth power. +* `floor(value)`: Rounds down. +* `log(n)`: Returns the natural logarithm. +* `log10(n)`: Returns the base 10 logarithm. +* `mod(dividend, divider)`: Returns the remainder. +* `pi()`: Returns the value of pi. +* `power(base, exponent)`: Returns the base raised to the power of the exponent. +* `radians(degrees)`: Returns the radians. +* `rand()`, `rand(seed)`: Returns a random number. +* `round(value, precision)`: Rounds to the specified decimal places. +* `sign(value)`: Returns the sign of the value. +* `sin(value)`: Returns the sine. +* `sqrt(value)`: Returns the square root. +* `tan(value)`: Returns the tangent. +* `truncate(numeric value, precision)`: Truncates the numeric value. + +#### **String Functions** + +* `concat(value1, value2)`: Concatenates two values. +* `lcase(string)`, `lower(string)`: Converts to lower case. +* `left(string, integer)`: Returns the left side of the string. +* `length(string)`: Returns the length. +* `locate(substring, string, [startIndex])`: Returns the location of a substring. +* `ltrim(string)`: Trims white space from the left. +* `repeat(string, count)`: Repeats the string. +* `rtrim(string)`: Trims white space from the right. +* `startswith(string, prefix)`: Tests if a string starts with a prefix. +* `substring(string, start, length)`: Returns a portion of the string. +* `ucase(string)`, `upper(string)`: Converts to upper case. + +#### **Date and Time Functions** + +* `age(date1, date2, [interval])`: Supplies the difference in age. +* `age_in_months(date1, date2)`: Returns age in months. +* `age_in_years(date1, date2)`: Returns age in years. +* `curdate()`, `curtime()`: Returns the current date/time. +* `dayofmonth(date)`: Returns the day of the month. +* `dayofweek(date)`: Returns the day of the week. +* `dayofyear(date)`: Returns the day of the year. +* `hour(time)`, `minute(time)`, `second(time)`: Return time components. +* `month(date)`, `monthname(date)`: Return month values. +* `now()`: Returns the system date and time. +* `quarter(date)`: Returns the yearly quarter. +* `timestampadd(interval, number, timestamp)`: Adds an interval. +* `timestampdiff(interval, ts1, ts2)`: Finds the difference between timestamps. +* `week(date)`, `year(date)`: Return week and year values. + +#### **Conditional and Utility Functions** + +* `coalesce(v1,...,vN)`: Returns the first non-null value. +* `greatest(a, b, c, ...)`: Returns the greatest value. +* `ifdefined(column_name)`: References columns that may not exist. +* `ifnull(testValue, defaultValue)`: Returns a default value if the test value is null. +* `isequal(a,b)`: Returns true if `a` equals `b` or if both are `NULL`. +* `least(a, b, c, ...)`: Returns the smallest value. + +#### **LabKey SQL Extensions** + +* `contextPath()`, `folderName()`, `folderPath()`: Return path information. +* `ismemberof(groupid)`: Checks if a user is a member of a group. +* `javaConstant(fieldName)`: Provides access to Java static final variables. +* `moduleProperty(module name, property name)`: Returns a module property. +* `overlaps(START1, END1, START2, END2)`: Tests for overlapping time intervals (PostgreSQL only). +* `userid()`, `username()`: Return user information. +* `version()`: Returns the current schema version. \ No newline at end of file diff --git a/query/src/org/labkey/query/controllers/QueryController.java b/query/src/org/labkey/query/controllers/QueryController.java index ee424e00487..52f6adda125 100644 --- a/query/src/org/labkey/query/controllers/QueryController.java +++ b/query/src/org/labkey/query/controllers/QueryController.java @@ -35,6 +35,7 @@ import org.apache.commons.collections4.MultiValuedMap; import org.apache.commons.collections4.multimap.ArrayListValuedHashMap; import org.apache.commons.collections4.multimap.HashSetValuedHashMap; +import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.Strings; import org.apache.commons.lang3.mutable.MutableInt; @@ -183,6 +184,7 @@ import org.labkey.api.query.QueryForm; import org.labkey.api.query.QueryParam; import org.labkey.api.query.QueryParseException; +import org.labkey.api.query.QueryParseWarning; import org.labkey.api.query.QuerySchema; import org.labkey.api.query.QueryService; import org.labkey.api.query.QuerySettings; @@ -232,6 +234,7 @@ import org.labkey.api.stats.BaseAggregatesAnalyticsProvider; import org.labkey.api.stats.ColumnAnalyticsProvider; import org.labkey.api.util.ButtonBuilder; +import org.labkey.api.util.ConfigurationException; import org.labkey.api.util.DOM; import org.labkey.api.util.ExceptionUtil; import org.labkey.api.util.FileUtil; @@ -8859,660 +8862,16 @@ public static class QueryAgentAction extends ReadOnlyApiAction { String getSQLHelp() { - // TODO use markdown translation instead of plain-text - return """ - This is documentation about LabKey SQL dialect: - - LabKey SQL - - LabKey SQL is a SQL dialect that supports (1) most standard SQL functionality and (2) provides extended functionality that is unique to LabKey, including: - - Security. Before execution, all SQL queries are checked against the user's security roles/permissions. - Lookup columns. Lookup columns use an intuitive syntax to access data in other tables to achieve what would normally require a JOIN statement. For example: "SomeTable.ForeignKey.FieldFromForeignTable" The special lookup column "Datasets" is injected into each study dataset and provides a syntax shortcut when joining the current dataset to another dataset that has compatible join keys. See example usage. - Cross-folder querying. Queries can be scoped to folders broader than the current folder and can draw from tables in folders other than the current folder. See Cross-Folder Queries. - Parameterized SQL statements. The PARAMETERS keyword lets you define parameters for a query. An associated API gives you control over the parameterized query from JavaScript code. See Parameterized SQL Queries. - Pivot tables. The PIVOT...BY and PIVOT...IN expressions provide a syntax for creating pivot tables. See Pivot Queries. - User-related functions. USERID() and ISMEMBEROF(groupid) let you control query visibility based on the user's group membership. - Ontology-related functions. (Premium Feature) Access preferred terms and ontology concepts from SQL queries. See Ontology SQL. - Lineage-related functions. (Premium Feature) Access ancestors and descendants of samples and data class entities. See Lineage SQL Queries. - Annotations. Override some column metadata using SQL annotations. See Use SQL Annotations. - - Reference Tables: - - Keywords - Constants - Operators - Operator Order of Precedence - Aggregate Functions - General - Aggregate Functions - PostgreSQL Only - SQL Functions - General - SQL Functions - PostgreSQL Specific - JSON and JSONB Operators and Functions - General Syntax - - Keywords - Keyword Description - AS Aliases can be explicitly named using the AS keyword. Note that the AS keyword is optional: the following select clauses both create an alias called "Name": - - SELECT LCASE(FirstName) AS Name - SELECT LCASE(FirstName) Name - - Implicit aliases are automatically generated for expressions in the SELECT list. In the query below, an output column named "Expression1" is automatically created for the expression "LCASE(FirstName)": - - SELECT LCASE(FirstName) - FROM PEOPLE - ASCENDING, ASC Return ORDER BY results in ascending value order. See the ORDER BY section for troubleshooting notes. - - ORDER BY Weight ASC - CAST(AS) CAST(R.d AS VARCHAR) - - Defined valid datatype keywords which can be used as cast/convert targets, and to what java.sql.Types name each keyword maps. Keywords are case-insensitive. - - BIGINT - BINARY - BIT - CHAR - DECIMAL - DATE - DOUBLE - FLOAT - GUID - INTEGER - LONGVARBINARY - LONGVARCHAR - NUMERIC - REAL - SMALLINT - TIME - TIMESTAMP - TINYINT - VARBINARY - VARCHAR - - Examples: - - CAST(TimeCreated AS DATE) - - CAST(WEEK(i.date) as INTEGER) as WeekOfYear, - - Precision and scale are supported when casting to NUMERIC. Example: - - CAST($Num AS NUMERIC(10,2)) - DESCENDING, DESC Return ORDER BY results in descending value order. See the ORDER BY section for troubleshooting notes. - - ORDER BY Weight DESC - DISTINCT Return distinct, non duplicate, values. - - SELECT DISTINCT Country - FROM Demographics - EXISTS Returns a Boolean value based on a subquery. Returns TRUE if at least one row is returned from the subquery. - - The following example returns any plasma samples which have been assayed with a score greater than 80%. Assume that ImmuneScores.Data.SpecimenId is a lookup field (aka foreign key) to Plasma.Name.; - - SELECT Plasma.Name - FROM Plasma - WHERE EXISTS - (SELECT * - FROM assay.General.ImmuneScores.Data - WHERE SpecimenId = Plasma.Name - AND ScorePercent > .8) - FALSE \s - FROM The FROM clause in LabKey SQL must contain at least one table. It can also contain JOINs to other tables. Commas are supported in the FROM clause: - - FROM TableA, TableB - WHERE TableA.x = TableB.x - - Nested joins are supported in the FROM clause: - - FROM TableA LEFT JOIN (TableB INNER JOIN TableC ON ...) ON... - - To refer to tables in LabKey folders other than the current folder, see Cross-Folder Queries. - GROUP BY Used with aggregate functions to group the results. Defines the "for each" or "per". The example below returns the number of records "for each" participant: - - SELECT ParticipantId, COUNT(Created) "Number of Records" - FROM "Physical Exam" - GROUP BY ParticipantId - HAVING Used with aggregate functions to limit the results. The following example returns participants with 10 or more records in the Physical Exam table: - - SELECT ParticipantId, COUNT(Created) "Number of Records" - FROM "Physical Exam" - GROUP BY ParticipantId - HAVING COUNT(Created) > 10 - - HAVING can be used without a GROUP BY clause, in which case all selected rows are treated as a single group for aggregation purposes. - JOIN, - RIGHT JOIN, - LEFT JOIN, - FULL JOIN, - CROSS JOIN Example: - - SELECT * - FROM "Physical Exam" - FULL JOIN "Lab Results" - ON "Physical Exam".ParticipantId = "Lab Results".ParticipantId - LIMIT Limits the number or records returned by the query. The following example returns the 10 most recent records: - - SELECT * - FROM "Physical Exam" - ORDER BY Created DESC LIMIT 10 - NULLIF(A,B) Returns NULL if A=B, otherwise returns A. - ORDER BY One option for sorting query results. It may produce unexpected results when dataregions or views also have sorting applied, or when using an expression in the ORDER BY clause, including an expression like table.columnName. If you can instead use a sort on the custom view or via, the API, those methods are preferred (see Troubleshooting note below). - - For best ORDER BY results, be sure to a) SELECT the columns on which you are sorting, b) sort on the SELECT column, not on an expression. To sort on an expression, include the expression in the SELECT (hidden if desired) and sort by the alias of the expression. For example: - - SELECT A, B, A+B AS C @hidden ... ORDER BY C - ...is preferable to: - SELECT A, B ... ORDER BY A+B - - Use ORDER BY with LIMIT to improve performance: - - SELECT ParticipantID, - Height_cm AS Height - FROM "Physical Exam" - ORDER BY Height DESC LIMIT 5 - - Troubleshooting: "Why is the ORDER BY clause not working as expected?" - - 1. Check to ensure you are sorting by a SELECT column (preferred) or an alias of an expression. Syntax like including the table name (i.e. ...ORDER BY table.columnName ASC) is an expression and should be aliased in the SELECT statement instead (i.e. SELECT table.columnName AS C ... ORDER BY C - - 2. When authoring queries in LabKey SQL, the query is typically processed as a subquery within a parent query. This parent query may apply it's own sorting overriding the ORDER BY clause in the subquery. This parent "view layer" provides default behavior like pagination, lookups, etc. but may also unexpectedly apply an additional sort. - - Two recommended solutions for more predictable sorting: - (A) Define the sort in the parent query using the grid view customizer. This may involve adding a new named view of that query to use as your parent query. - (B) Use the "sort" property in the selectRows API call. - PARAMETERS Queries can declare parameters using the PARAMETERS keyword. Default values data types are supported as shown below: - - PARAMETERS (X INTEGER DEFAULT 37) - SELECT * - FROM "Physical Exam" - WHERE Temp_C = X - - Parameter names will override any unqualified table column with the same name. Use a table qualification to disambiguate. In the example below, R.X refers to the column while X refers to the parameter: - - PARAMETERS(X INTEGER DEFAULT 5) - SELECT * - FROM Table R - WHERE R.X = X - - Supported data types for parameters are: BIGINT, BIT, CHAR, DECIMAL, DOUBLE, FLOAT, INTEGER, LONGVARCHAR, NUMERIC, REAL, SMALLINT, TIMESTAMP, TINYINT, VARCHAR - - Numeric parameters can include precision and scale: - - PARAMETERS($NUM NUMERIC(10,2)) - - Parameter values can be passed via JavaScript API calls to the query. For details see Parameterized SQL Queries. - PIVOT/PIVOT...BY/PIVOT...IN Re-visualize a table by rotating or "pivoting" a portion of it, essentially promoting cell data to column headers. See Pivot Queries for details and examples. - SELECT SELECT queries are the only type of query that can currently be written in LabKey SQL. Sub-selects are allowed both as an expression, and in the FROM clause. - - Aliases are automatically generated for expressions after SELECT. In the query below, an output column named "Expression1" is automatically generated for the expression "LCASE(FirstName)": - - SELECT LCASE(FirstName) FROM... - TRUE \s - UNION, UNION ALL The UNION clause is the same as standard SQL. LabKey SQL supports UNION in subqueries. - VALUES ... AS A subset of VALUES syntax is supported. Generate a "constant table" by providing a parenthesized list of expressions for each row in the table. The lists must all have the same number of elements and corresponding entries must have compatible data types. For example: - - VALUES (1, 'one'), (2, 'two'), (3, 'three') AS t; - You must provide the alias for the result ("AS t" in the above), aliasing column names is not supported. The column names will be 'column1', 'column2', etc. - WHERE Filter the results for certain values. Example: - - SELECT * - FROM "Physical Exam" - WHERE YEAR(Date) = 2010 - WITH - - Define a "common table expression" which functions like a subquery or inline view table. Especially useful for recursive queries. - - Usage Notes: If there are UNION clauses that do not reference the common table expression (CTE) itself, the server interprets them as normal UNIONs. The first subclause of a UNION may not reference the CTE. The CTE may only be referenced once in a FROM clause or JOIN clauses within the UNION. There may be multiple CTEs defined in the WITH. Each may reference the previous CTEs in the WITH. No column specifications are allowed in the WITH (as some SQL versions allow). - - Exception Behavior: Testing indicates that PostgreSQL does not provide an exception to LabKey Server for a non-ending, recursive CTE query. This can cause the LabKey Server to wait indefinitely for the query to complete. - - A non-recursive example: - - WITH AllDemo AS - ( - SELECT * - FROM "/Studies/Study A/".study.Demographics - UNION - SELECT * - FROM "/Studies/Study B/".study.Demographics - ) - SELECT ParticipantId from AllDemo - - A recursive example: In a table that holds parent/child information, this query returns all of the children and grandchildren (recursively down the generations), for a given "Source" parent. - - PARAMETERS - ( - Source VARCHAR DEFAULT NULL - ) - - WITH Derivations AS\s - (\s - -- Anchor Query. User enters a 'Source' parent\s - SELECT Item, Parent - FROM Items\s - WHERE Parent = Source - UNION ALL\s - - -- Recursive Query. Get the children, grandchildren, ... of the source parent - SELECT i.Item, i.Parent\s - FROM Items i INNER JOIN Derivations p\s - ON i.Parent = p.Item\s - )\s - SELECT * FROM Derivations - - - Constants - - The following constant values can be used in LabKey SQL queries. - Constant Description - CAST('Infinity' AS DOUBLE) Represents positive infinity. - CAST('-Infinity' AS DOUBLE) Represents negative infinity. - CAST('NaN' AS DOUBLE) Represents "Not a number". - TRUE Boolean value. - FALSE Boolean value. - - Operators - Operator Description - String Operators Note that strings are delimited with single quotes. Double quotes are used for column and table names containing spaces. - || String concatenation. For example: - - SELECT ParticipantId, - City || ', ' || State AS CityOfOrigin - FROM Demographics - - If any argument is null, the || operator will return a null string. To handle this, use COALESCE with an empty string as it's second argument, so that the other || arguments will be returned: - City || ', ' || COALESCE (State, '') - LIKE Pattern matching. The entire string must match the given pattern. Ex: LIKE 'W%'. - NOT LIKE Negative pattern matching. Will return values that do not match a given pattern. Ex: NOT LIKE 'W%' - Arithmetic Operators \s - + Add - - Subtract - * Multiply - / Divide - Comparison operators \s - = Equals - != Does not equal - <> Does not equal - > Is greater than - < Is less than - >= Is greater than or equal to - <= Is less than or equal to - IS NULL Is NULL - IS NOT NULL Is NOT NULL - BETWEEN Between two values. Values can be numbers, strings or dates. - IN Example: WHERE City IN ('Seattle', 'Portland') - NOT IN Example: WHERE City NOT IN ('Seattle', 'Portland') - Bitwise Operators \s - & Bitwise AND - | Bitwise OR - ^ Bitwise exclusive OR - Logical Operators \s - AND Logical AND - OR Logical OR - NOT Example: WHERE NOT Country='USA' - Operator Order of Precedence - Order of Precedence Operators - 1 - (unary) , + (unary), CASE - 2 *, / (multiplication, division) - 3 +, -, & (binary plus, binary minus) - 4 & (bitwise and) - 5 ^ (bitwise xor) - 6 | (bitwise or) - 7 || (concatenation) - 8 <, >, <=, >=, IN, NOT IN, BETWEEN, NOT BETWEEN, LIKE, NOT LIKE - 9 =, IS, IS NOT, <>, != - 10 NOT - 11 AND - 12 OR - Aggregate Functions - General - Function Description - COUNT The special syntax COUNT(*) is supported as of LabKey v9.2. - MIN Minimum - MAX Maximum - AVG Average - SUM Sum\s - GROUP_CONCAT An aggregate function, much like MAX, MIN, AVG, COUNT, etc. It can be used wherever the standard aggregate functions can be used, and is subject to the same grouping rules. It will return a string value which is comma-separated list of all of the values for that grouping. A custom separator, instead of the default comma, can be specified. Learn more here. The example below specifies a semi-colon as the separator: - - SELECT Participant, GROUP_CONCAT(DISTINCT Category, ';') AS CATEGORIES FROM SomeSchema.SomeTable - - To use a line-break as the separator, use the following: - - SELECT Participant, GROUP_CONCAT(DISTINCT Category, chr(10)) AS CATEGORIES FROM SomeSchema.SomeTable - stddev(expression) Standard deviation - stddev_pop(expression) Population standard deviation of the input values. - variance(expression) Historical alias for var_samp. - var_pop(expression) Population variance of the input values (square of the population standard deviation). - median(expression) The 50th percentile of the values submitted. - Aggregate Functions - PostgreSQL Only - Function Description - bool_and(expression) Aggregates boolean values. Returns true if all values are true and false if any are false. - bool_or(expression) Aggregates boolean values. Returns true if any values are true and false if all are false. - bit_and(expression) Returns the bitwise AND of all non-null input values, or null if none. - bit_or(expression) Returns the bitwise OR of all non-null input values, or null if none. - every(expression) Equivalent to bool_and(). Returns true if all values are true and false if any are false. - corr(Y,X) Correlation coefficient. - covar_pop(Y,X) Population covariance. - covar_samp(Y,X) Sample covariance. - regr_avgx(Y,X) Average of the independent variable: (SUM(X)/N). - regr_avgy(Y,X) Average of the dependent variable: (SUM(Y)/N). - regr_count(Y,X) Number of non-null input rows. - regr_intercept(Y,X) Y-intercept of the least-squares-fit linear equation determined by the (X,Y) pairs. - regr_r2(Y,X) Square of the correlation coefficient. - regr_slope(Y,X) Slope of the least-squares-fit linear equation determined by the (X,Y) pairs. - regr_sxx(Y,X) Sum of squares of the independent variable. - regr_sxy(Y,X) Sum of products of independent times dependent variable. - regr_syy(Y,X) Sum of squares of the dependent variable. - stddev_samp(expression) Sample standard deviation of the input values. - var_samp(expression) Sample variance of the input values (square of the sample standard deviation). - SQL Functions - General - - Many of these functions are similar to standard SQL functions -- see the JBDC escape syntax documentation. - Function Description - abs(value) Returns the absolute value. - acos(value) Returns the arc cosine. - age(date1, date2) - - Supplies the difference in age between the two dates, calculated in years. - age(date1, date2, interval) - - The interval indicates the unit of age measurement, either SQL_TSI_MONTH or SQL_TSI_YEAR. - age_in_months(date1, date2) Behavior is undefined if date2 is before date1. - age_in_years(date1, date2) Behavior is undefined if date2 is before date1. - asin(value) Returns the arc sine. - atan(value) Returns the arc tangent. - atan2(value1, value2) Returns the arctangent of the quotient of two values. - case - - CASE can be used to test various conditions and return various results based on the test. You can use either simple CASE or searched CASE syntax. In the following examples "value#" indicates a value to match against, where "test#" indicates a boolean expression to evaluate. In the "searched" syntax, the first test expression that evaluates to true will determine which result is returned. Note that the LabKey SQL parser sometimes requires the use of additional parentheses within the statement. - - CASE (value) WHEN (value1) THEN (result1) ELSE (result2) END - CASE (value) WHEN (value1) THEN (result1) WHEN (value2) THEN (result2) ELSE (resultDefault) END - CASE WHEN (test1) THEN (result1) ELSE (result2) END - CASE WHEN (test1) THEN (result1) WHEN (test2) THEN (result2) WHEN (test3) THEN (result3) ELSE (resultDefault) END - - Example: - - SELECT "StudentName", - School, - CASE WHEN (Division = 'Grades 3-5') THEN (Scores.Score*1.13) ELSE Score END AS AdjustedScore, - Division - FROM Scores - ceiling(value) Rounds the value up. - coalesce(value1,...,valueN) Returns the first non-null value in the argument list. Use to set default values for display. - concat(value1,value2) Concatenates two values. - contextPath() Returns the context path starting with “/” (e.g. “/labkey”). Returns the empty string if there is no current context path. (Returns VARCHAR.) - cos(radians) Returns the cosine. - cot(radians) Returns the cotangent. - curdate() Returns the current date. - curtime() Returns the current time - dayofmonth(date) Returns the day of the month (1-31) for a given date. - dayofweek(date) Returns the day of the week (1-7) for a given date. (Sun=1 and Sat=7) - dayofyear(date) Returns the day of the year (1-365) for a given date. - degrees(radians) Returns degrees based on the given radians. - exp(n) Returns Euler's number e raised to the nth power. e = 2.71828183 - floor(value) Rounds down to the nearest integer. - folderName() LabKey SQL extension function. Returns the name of the current folder, without beginning or trailing "/". (Returns VARCHAR.) - folderPath() LabKey SQL extension function. Returns the current folder path (starts with “/”, but does not end with “/”). The root returns “/”. (Returns VARCHAR.) - greatest(a, b, c, ...) Returns the greatest value from the list expressions provided. Any number of expressions may be used. The expressions must have the same data type, which will also be the type of the result. The LEAST() function is similar, but returns the smallest value from the list of expressions. GREATEST() and LEAST() are not implemented for SAS databases. - - When NULL values appear in the list of expressions, different database implementations as follows: - - - PostgreSQL & MS SQL Server ignore NULL values in the arguments, only returning NULL if all arguments are NULL. - - Oracle and MySql return NULL if any one of the arguments are NULL. Best practice: wrap any potentially nullable arguments in coalesce() or ifnull() and determine at the time of usage if NULL should be treated as high or low. - - Example: - - SELECT greatest(score_1, score_2, score_3) As HIGH_SCORE - FROM MyAssay - hour(time) Returns the hour for a given date/time. - ifdefined(column_name) IFDEFINED(NAME) allows queries to reference columns that may not be present on a table. Without using IFDEFINED(), LabKey will raise a SQL parse error if the column cannot be resolved. Using IFDEFINED(), a column that cannot be resolved is treated as a NULL value. The IFDEFINED() syntax is useful for writing queries over PIVOT queries or assay tables where columns may be added or removed by an administrator. - ifnull(testValue, defaultValue) If testValue is null, returns the defaultValue. Example: IFNULL(Units,0) - isequal LabKey SQL extension function. ISEQUAL(a,b) is equivalent to (a=b OR (a IS NULL AND b IS NULL)) - ismemberof(groupid) LabKey SQL extension function. Returns true if the current user is a member of the specified group. - javaConstant(fieldName) LabKey SQL extension function. Provides access to public static final variable values. For details see LabKey SQL Utility Functions. - lcase(string) Convert all characters of a string to lower case. - least(a, b, c, ...) Returns the smallest value from the list expressions provided. For more details, see greatest() above. - left(string, integer) Returns the left side of the string, to the given number of characters. Example: SELECT LEFT('STRINGVALUE',3) returns 'STR' - length(string) Returns the length of the given string. - locate(substring, string) locate(substring, string, startIndex) Returns the location of the first occurrence of substring within string. startIndex provides a starting position to begin the search. - log(n) Returns the natural logarithm of n. - log10(n) Base base 10 logarithm on n. - lower(string) Convert all characters of a string to lower case. - ltrim(string) Trims white space characters from the left side of the string. For example: LTRIM(' Trim String') - minute(time) Returns the minute value for the given time. - mod(dividend, divider) Returns the remainder of the division of dividend by divider. - moduleProperty(module name, property name) - - LabKey SQL extension function. Returns a module property, based on the module and property names. For details see LabKey SQL Utility Functions. - month(date) Returns the month value (1-12) of the given date. - monthname(date) Return the month name of the given date. - now() Returns the system date and time. - overlaps LabKey SQL extension function. Supported only when PostgreSQL is installed as the primary database. \s - - SELECT OVERLAPS (START1, END1, START2, END2) AS COLUMN1 FROM MYTABLE - - The LabKey SQL syntax above is translated into the following PostgreSQL syntax: \s - - SELECT (START1, END1) OVERLAPS (START2, END2) AS COLUMN1 FROM MYTABLE - pi() Returns the value of pi;. - power(base, exponent) Returns the base raised to the power of the exponent. For example, power(10,2) returns 100. - quarter(date) Returns the yearly quarter for the given date where the 1st quarter = Jan 1-Mar 31, 2nd quarter = Apr 1-Jun 30, 3rd quarter = Jul 1-Sep 30, 4th quarter = Oct 1-Dec 31 - radians(degrees) Returns the radians for the given degrees. - rand(), rand(seed) Returns a random number between 0 and 1. - repeat(string, count) Returns the string repeated the given number of times. SELECT REPEAT('Hello',2) returns 'HelloHello'. - round(value, precision) Rounds the value to the specified number of decimal places. ROUND(43.3432,2) returns 43.34 - rtrim(string) Trims white space characters from the right side of the string. For example: RTRIM('Trim String ') - second(time) Returns the second value for the given time. - sign(value) Returns the sign, positive or negative, for the given value. - sin(value) Returns the sine for the given value. - startswith(string, prefix) Tests to see if the string starts with the specified prefix. For example, STARTSWITH('12345','2') returns FALSE. - sqrt(value) Returns the square root of the value. - substring(string, start, length) Returns a portion of the string as specified by the start location (1-based) and length (number of characters). For example, substring('SomeString', 1,2) returns the string 'So'. - tan(value) - - Returns the tangent of the value. - timestampadd(interval, number_to_add, timestamp) - - Adds an interval to the given timestamp value. The interval value must be surrounded by quotes. Possible values for interval: - - SQL_TSI_FRAC_SECOND - SQL_TSI_SECOND - SQL_TSI_MINUTE - SQL_TSI_HOUR - SQL_TSI_DAY - SQL_TSI_WEEK - SQL_TSI_MONTH - SQL_TSI_QUARTER - SQL_TSI_YEAR - - Example: TIMESTAMPADD('SQL_TSI_QUARTER', 1, "Physical Exam".date) AS NextExam - timestampdiff(interval, timestamp1, timestamp2) - - Finds the difference between two timestamp values at a specified interval. The interval must be surrounded by quotes. - - Example: TIMESTAMPDIFF('SQL_TSI_DAY', SpecimenEvent.StorageDate, SpecimenEvent.ShipDate) - - Note that PostgreSQL does not support the following intervals: - - SQL_TSI_FRAC_SECOND - SQL_TSI_WEEK - SQL_TSI_MONTH - SQL_TSI_QUARTER - SQL_TSI_YEAR - - As a workaround, use the 'age' functions defined above. - truncate(numeric value, precision) Truncates the numeric value to the precision specified. This is an arithmetic truncation, not a string truncation. - TRUNCATE(123.4567,1) returns 123.4 - TRUNCATE(123.4567,2) returns 123.45 - TRUNCATE(123.4567,-1) returns 120.0 - - May require an explict CAST into NUMERIC, as LabKey SQL does not check data types for function arguments. - - SELECT - PhysicalExam.Temperature, - TRUNCATE(CAST(Temperature AS NUMERIC),1) as truncTemperature - FROM PhysicalExam - ucase(string), upper(string) Converts all characters to upper case. - userid() LabKey SQL extension function. Returns the userid, an integer, of the logged in user. - username() LabKey SQL extension function. Returns the current user display name. VARCHAR - version() LabKey SQL extension function. Returns the current schema version of the core module as a NUMERIC with four decimal places. For example: 20.0070 - week(date) Returns the week value (1-52) of the given date. - year(date) Return the year of the given date. Assuming the system date is March 4 2023, then YEAR(NOW()) return 2023. - SQL Functions - PostgreSQL Specific - - LabKey SQL supports the following PostgreSQL functions. - See the PostgreSQL docs for usage details. - PostgreSQL Function Docs\s - ascii(value) Returns the ASCII code of the first character of value. \s - btrim(value, - trimchars) Removes characters in trimchars from the start and end of string. trimchars defaults to white space. - - BTRIM(' trim ') returns TRIM\s - BTRIM('abbatrimabtrimabba', 'ab') returns trimabtrim - - character_length(value), char_length(value) - Returns the number of characters in value. - chr(integer_code) Returns the character with the given integer_code. - - CHR(70) returns F - concat_ws(sep text, - val1 "any" [, val2 "any" [,...]]) -> text Concatenates all but the first argument, with separators. The first argument is used as the separator string, and should not be NULL. Other NULL arguments are ignored. See the PostgreSQL docs. - - concat_ws(',', 'abcde', 2, NULL, 22) → abcde,2,22 - decode(text, - format) See the PostgreSQL docs. - encode(binary, - format) See the PostgreSQL docs. - is_distinct_from(a, b) OR - is_not_distinct_from(a, b) Not equal (or equal), treating null like an ordinary value. - initcap(string) Converts the first character of each separate word in string to uppercase and the rest to lowercase. - lpad(string,\s - int, - fillchars) Pads string to length int by prepending characters fillchars. - md5(text) Returns the hex MD5 value of text. - octet_length(string) Returns the number of bytes in string. - overlaps See above for syntax details. - quote_ident(string) Returns string quoted for use as an identifier in an SQL statement. - quote_literal(string) Returns string quoted for use as a string literal in an SQL statement. - regexp_replace See PostgreSQL docs for details: reference doc, example doc - repeat(string, int) Repeats string the specified number of times. - replace(string,\s - matchString,\s - replaceString) Searches string for matchString and replaces occurrences with replaceString. - rpad(string,\s - int, - fillchars) Pads string to length int by postpending characters fillchars. - similar_to(A,B,C) String pattern matching using SQL regular expressions. 'A' similar to 'B' escape 'C'. See the PostgreSQL docs. - split_part(string, - delimiter, - int) Splits string on delimiter and returns fragment number int (starting the count from 1). - - SPLIT_PART('mississippi', 'i', 4) returns 'pp'. - string_to_array See Array Functions in the PostgreSQL docs. - strpos(string, - substring) Returns the position of substring in string. (Count starts from 1.) - substr(string, - fromPosition, - charCount) - - Extracts the number of characters specified by charCount from string starting at position fromPosition. - - SUBSTR('char_sequence', 5, 2) returns '_s'\s - to_ascii(string, - encoding) Convert string to ASCII from another encoding. - to_hex(int) Converts int to its hex representation. - translate(text, - fromText, - toText) Characters in string matching a character in the fromString set are replaced by the corresponding character in toString. - to_char See Data Type Formatting Functions in the PostgreSQL docs. - to_date(textdate, - format) See Data Type Formatting Functions in the PostgreSQL docs.\s - to_timestamp See Data Type Formatting Functions in the PostgreSQL docs. - to_number See Data Type Formatting Functions in the PostgreSQL docs. - unnest See Array Functions in the PostgreSQL docs. - JSON and JSONB Operators and Functions - - LabKey SQL supports the following PostgreSQL JSON and JSONB operators and functions. Note that LabKey SQL does not natively understand arrays and some other features, but it may still be possible to use the functions that expect them. - See the PostgreSQL docs for usage details. - PostgreSQL Operators and Functions Docs - ->, ->>, #>, #>>, @>, <@, ?, ?|, ?&, ||, -, #- - LabKey SQL supports these operators via a pass-through function, json_op. The function's first argument is the operator's first operand. The first second is the operator, passed as a string constant. The function's third argument is the second operand. For example, this Postgres SQL expression: - - a_jsonb_column --> 2 - - can be represented in LabKey SQL as: - - json_op(a_jsonb_column, '-->', 2) - - parse_json, parse_jsonb Casts a text value to a parsed JSON or JSONB data type. For example, - - '{"a":1, "b":null}'::jsonb - - or - - CAST('{"a":1, "b":null}' AS JSONB) - - can be represented in LabKey SQL as: - - parse_jsonb('{"a":1, "b":null}') - - to_json, to_jsonb Converts a value to the JSON or JSONB data type. Will treat a text value as a single JSON string value - array_to_json Converts an array value to the JSON data type. - row_to_json Converts a scalar (simple value) row to JSON. Note that LabKey SQL does not support the version of this function that will convert an entire table to JSON. Consider using "to_jsonb()" instead. - json_build_array, jsonb_build_array Build a JSON array from the arguments - json_build_object, jsonb_build_object Build a JSON object from the arguments - json_object, jsonb_object Build a JSON object from a text array - json_array_length, jsonb_array_length Return the length of the outermost JSON array - json_each, jsonb_each Expand the outermost JSON object into key/value pairs. Note that LabKey SQL does not support the table version of this function. Usage as a scalar function like this is supported: - - SELECT json_each('{"a":"foo", "b":"bar"}') AS Value - - json_each_text, jsonb_each_text Expand the outermost JSON object into key/value pairs into text. Note that LabKey SQL does not support the table version of this function. Usage as a scalar function (similar to json_each) is supported. - json_extract_path, jsonb_extract_path Return the JSON value referenced by the path - json_extract_path_text, jsonb_extract_path_text Return the JSON value referenced by the path as text - json_object_keys, jsonb_object_keys Return the keys of the outermost JSON object - json_array_elements, jsonb_array_elements Expand a JSON array into a set of values - json_array_elements_text, jsonb_array_elements_text Expand a JSON array into a set of text values - json_typeof, jsonb_typeof Return the type of the outermost JSON object - json_strip_nulls, jsonb_strip_nulls Remove all null values from a JSON object - jsonb_insert Insert a value within a JSON object at a given path - jsonb_pretty Format a JSON object as indented text - jsonb_set Set the value within a JSON object for a given path. Strict, i.e. returns NULL on NULL input. - jsonb_set_lax Set the value within a JSON object for a given path. Not strict; expects third argument to specify how to treat NULL input (one of 'raise_exception', 'use_json_null', 'delete_key', or 'return_target'). - jsonb_path_exists, jsonb_path_exists_tz Checks whether the JSON path returns any item for the specified JSON value. The "_tz" variant is timezone aware. - jsonb_path_match, jsonb_path_match_tz Returns the result of a JSON path predicate check for the specified JSON value. The "_tz" variant is timezone aware. - jsonb_path_query, jsonb_path_query_tz Returns all JSON items returned by the JSON path for the specified JSON value. The "_tz" variant is timezone aware. - jsonb_path_query_array, jsonb_path_query_array_tz Returns as an array, all JSON items returned by the JSON path for the specified JSON value. The "_tz" variant is timezone aware. - jsonb_path_query_first, jsonb_path_query_first_tz Returns the first JSON item returned by the JSON path for the specified JSON value. The "_tz" variant is timezone aware. - General Syntax - Syntax Item Description - Case Sensitivity Schema names, table names, column names, SQL keywords, function names are case-insensitive in LabKey SQL. - Comments Comments that use the standard SQL syntax can be included in queries. '--' starts a line comment. Also, '/* */' can surround a comment block: - - -- line comment 1 - -- line comment 2 - /* block comment 1 - block comment 2 */ - SELECT ...\s - Identifiers Identifiers in LabKey SQL may be quoted using double quotes. (Double quotes within an identifier are escaped with a second double quote.) - - SELECT "Physical Exam".* - ...\s - Lookups Lookups columns reference data in other tables. In SQL terms, they are foreign key columns. See Lookups for details on creating lookup columns. Lookups use a convenient syntax of the form "Table.ForeignKey.FieldFromForeignTable" to achieve what would normally require a JOIN in SQL. Example: - - Issues.AssignedTo.DisplayName - String Literals String literals are quoted with single quotes ('). Within a single quoted string, a single quote is escaped with another single quote. - - SELECT * FROM TableName WHERE FieldName = 'Jim''s Item'\s - Date/Time Literals - - Date and Timestamp (Date&Time) literals can be specified using the JDBC escape syntax - - {ts '2001-02-03 04:05:06'} - - {d '2001-02-03'}\s - """; + try + { + return IOUtils.resourceToString("org/labkey/query/controllers/LabKeySql.md", null, QueryController.class.getClassLoader()); + } + catch (IOException x) + { + throw new ConfigurationException("error loading resource", x); + } } + Content contentFromText(String s) { return Content.fromParts(Part.fromText(s)); @@ -9555,7 +8914,7 @@ Chat getChat(String currentSchema) Chat chatSession = client.chats.create(getModel(), config); StringBuilder serviceMessage = new StringBuilder(); - serviceMessage.append("Your job is to generate SQL statements. Here is some reference material you can use:\n\n").append(getSQLHelp()); + serviceMessage.append("Your job is to generate SQL statements. Here is some reference material formatted as markdown:\n").append(getSQLHelp()).append("\n"); QuerySchema defaultSchema = DefaultSchema.get(getUser(), getContainer()); StringBuilder sb = new StringBuilder(); @@ -9609,10 +8968,17 @@ public Object execute(PromptForm form, BindException errors) throws Exception try { TableInfo ti = QueryService.get().createTable(schema, sql, null, true); + var warnings = ti.getWarnings(); + if (null != warnings) + { + var warning = warnings.stream().findFirst(); + if (warning.isPresent()) + throw warning.get(); + } } catch (QueryException x) { - String validationPrompt = "That SQL caused the error below, can you attempt to fix this?\n```" + x.getMessage() + "```"; + String validationPrompt = "That SQL caused the " + (x instanceof QueryParseWarning ? "warning" : "error") + " below, can you attempt to fix this?\n```" + x.getMessage() + "```"; response = chatSession.sendMessage(validationPrompt); text = response.text(); var newSQL = extractSql(response.text()); From 1be5fd71f3a3f4eb25d1c3648c5766cd505cdab7 Mon Sep 17 00:00:00 2001 From: labkey-matthewb Date: Tue, 16 Sep 2025 10:42:22 -0700 Subject: [PATCH 04/25] comment out console.log --- query/src/org/labkey/query/view/sourceQuery.jsp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/query/src/org/labkey/query/view/sourceQuery.jsp b/query/src/org/labkey/query/view/sourceQuery.jsp index 2ff8af896ac..13e5219fcda 100644 --- a/query/src/org/labkey/query/view/sourceQuery.jsp +++ b/query/src/org/labkey/query/view/sourceQuery.jsp @@ -256,8 +256,8 @@ Ext4.onReady(function(){ const resizeFn = function(evt) { - console.log(evt); - console.log("window " + window.innerHeight + " " + window.innerWidth); + // console.log(evt); + // console.log("window " + window.innerHeight + " " + window.innerWidth); var el = document.getElementById("querySourceLayout"); if (el) { From 61746b2ae890bdeedd2b9a72bb49b856a36aaa60 Mon Sep 17 00:00:00 2001 From: labkey-matthewb Date: Wed, 17 Sep 2025 09:35:33 -0700 Subject: [PATCH 05/25] include saved queries --- .../query/controllers/QueryController.java | 66 ++++++++++++++----- 1 file changed, 48 insertions(+), 18 deletions(-) diff --git a/query/src/org/labkey/query/controllers/QueryController.java b/query/src/org/labkey/query/controllers/QueryController.java index 52f6adda125..24c42ab36d4 100644 --- a/query/src/org/labkey/query/controllers/QueryController.java +++ b/query/src/org/labkey/query/controllers/QueryController.java @@ -19,6 +19,7 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.genai.errors.ServerException; import com.google.genai.types.FunctionDeclaration; import com.google.genai.types.GenerateContentConfig; import com.google.genai.types.Part; @@ -8895,7 +8896,7 @@ Chat getChat(String currentSchema) .build())) .required("quotedTableName") .build(); - var listColumnMetaDataFn = FunctionDeclaration.builder().name("listColumnsForTable").description("Provide column metadata for a sql table.").parameters(columnsMetaDataParameters); + var listColumnMetaDataFn = FunctionDeclaration.builder().name("listColumnsForTable").description("Provide column metadata for a sql table. This tool Will also return SQL source for saved queries.").parameters(columnsMetaDataParameters); Schema tablesMetaDataParameters = Schema.builder() .type(Type.Known.OBJECT) @@ -8905,7 +8906,7 @@ Chat getChat(String currentSchema) .build())) .required("quotedSchemaName") .build(); - var listTablesMetaDataFn = FunctionDeclaration.builder().name("listTablesForSchema").description("Provide column metadata for a sql table.").parameters(tablesMetaDataParameters); + var listTablesMetaDataFn = FunctionDeclaration.builder().name("listTablesForSchema").description("Provide column metadata for a database table.").parameters(tablesMetaDataParameters); var tools = Tool.builder().functionDeclarations(listColumnMetaDataFn, listTablesMetaDataFn); @@ -8954,7 +8955,19 @@ public Object execute(PromptForm form, BindException errors) throws Exception String prompt = form.getPrompt(); for (int retry=0 ; retry < 5 ; retry++) { - GenerateContentResponse response = chatSession.sendMessage(prompt); + GenerateContentResponse response; + try + { + response = chatSession.sendMessage(prompt); + } + catch (ServerException x) + { + return new JSONObject(Map.of( + "model", getModel(), + "error", x.getMessage(), + "text", "ERROR: " + x.getMessage(), + "success", Boolean.FALSE)); + } var functionCalls = response.functionCalls(); if (null == functionCalls || functionCalls.isEmpty()) { @@ -9084,23 +9097,29 @@ public static JSONObject listTablesForSchema(String fullQuotedName) ViewContext context = HttpView.currentView().getViewContext(); var defaultSchema = DefaultSchema.get(context.getUser(), context.getContainer()); var schema = DefaultSchema.resolve(defaultSchema, fullKey); - if (null == schema) + if (!(schema instanceof UserSchema userSchema)) return new JSONObject("error", "could not find schema for : " + fullQuotedName); JSONArray array = new JSONArray(); - for (String tableName : schema.getTableNames()) - { - // CONSIDER schema.getTableDescription()??? - TableInfo td = schema.getTable(tableName, null); - if (null == td) - continue; - JSONObject table = new JSONObject(); - table.put("schemaName", schema.getName()); - table.put("tableName", td.getName()); - table.put("fullQuotedName", new SchemaKey(schema.getSchemaPath(), td.getName()).toSQLString()); - table.put("description", td.getDescription()); - array.put(table); - } + CaseInsensitiveHashSet names = new CaseInsensitiveHashSet(schema.getTableNames()); + var qds = userSchema.getQueryDefs(); + names.addAll(qds.keySet()); + + for (String tableName : names) + { + // CONSIDER schema.getTableDescription()??? + TableInfo td = schema.getTable(tableName, null); + if (null == td) + continue; + QueryDefinition qd = ((UserSchema)schema).getQueryDef(tableName); + JSONObject table = new JSONObject(); + table.put("schemaName", schema.getName()); + table.put("tableName", td.getName()); + table.put("fullQuotedName", new SchemaKey(schema.getSchemaPath(), td.getName()).toSQLString()); + table.put("description", td.getDescription()); + table.put("type", null==qd ? "TABLE" : "QUERY"); + array.put(table); + } return new JSONObject(Map.of("tables", array)); } @@ -9149,11 +9168,22 @@ else if (fullKey.size() == 1) if (null == td) return new JSONObject("error", "could not find table"); + String sourceSQL = null; + if (schema instanceof UserSchema userSchema) + { + QueryDefinition d = userSchema.getQueryDef(tableName); + if (null != d) + sourceSQL = d.getSql(); + } + JSONObject table = new JSONObject(); table.put("schemaName", schema.getName()); table.put("tableName", td.getName()); table.put("fullQuotedName", new SchemaKey(schema.getSchemaPath(), td.getName()).toSQLString()); - table.put("description", td.getDescription()); + if (isNotBlank(td.getDescription())) + table.put("description", td.getDescription()); + if (isNotBlank(sourceSQL)) + table.put("sql", sourceSQL); var pkColumns = td.getPkColumns(); var pk = pkColumns.size() == 1 ? pkColumns.get(0).getFieldKey() : null; From 5dfe50c57ae40cca518d4e062ce3676e580db924 Mon Sep 17 00:00:00 2001 From: Matthew Bellew Date: Wed, 17 Sep 2025 13:25:15 -0700 Subject: [PATCH 06/25] Nested Schema --- .../query/controllers/QueryController.java | 69 ++++++++++++++++--- 1 file changed, 58 insertions(+), 11 deletions(-) diff --git a/query/src/org/labkey/query/controllers/QueryController.java b/query/src/org/labkey/query/controllers/QueryController.java index 24c42ab36d4..b24f2de6f55 100644 --- a/query/src/org/labkey/query/controllers/QueryController.java +++ b/query/src/org/labkey/query/controllers/QueryController.java @@ -348,6 +348,7 @@ import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.TreeMap; import java.util.TreeSet; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -364,6 +365,7 @@ import static org.labkey.api.action.ApiJsonWriter.CONTENT_TYPE_JSON; import static org.labkey.api.assay.AssayFileWriter.ensureUploadDirectory; import static org.labkey.api.data.DbScope.NO_OP_TRANSACTION; +import static org.labkey.api.data.views.DataViewProvider.EditInfo.Property.name; import static org.labkey.api.query.AbstractQueryUpdateService.saveFile; import static org.labkey.api.util.DOM.BR; import static org.labkey.api.util.DOM.DIV; @@ -8916,19 +8918,17 @@ Chat getChat(String currentSchema) StringBuilder serviceMessage = new StringBuilder(); serviceMessage.append("Your job is to generate SQL statements. Here is some reference material formatted as markdown:\n").append(getSQLHelp()).append("\n"); + serviceMessage.append("NOTE: please prefer using lookup syntax rather than JOIN where possible.\n"); - QuerySchema defaultSchema = DefaultSchema.get(getUser(), getContainer()); + DefaultSchema defaultSchema = DefaultSchema.get(getUser(), getContainer()); + Map schemaMap = listAllSchemas(defaultSchema); StringBuilder sb = new StringBuilder(); - for (var name : defaultSchema.getSchemaNames()) + for (var schema : schemaMap.values()) { - var schema = defaultSchema.getSchema(name); - if (null != schema) - { - sb.append("\t* ").append(schema.getSchemaPath().toSQLString()); - if (isNotBlank(schema.getDescription())) - sb.append("\t").append(schema.getDescription()); - sb.append("\n"); - } + sb.append("\t* ").append(schema.getSchemaPath().toSQLString()); + if (isNotBlank(schema.getDescription())) + sb.append("\t").append(schema.getDescription()); + sb.append("\n"); } serviceMessage.append("\n\nHere are the available schemas:\n" + sb); @@ -9078,6 +9078,46 @@ public Object execute(Object o, BindException errors) throws Exception } } + + /* For now, list all schemas. CONSIDER support incremental querying. */ + public static Map listAllSchemas(DefaultSchema root) + { + SimpleSchemaTreeVisitor, Void> visitor = new SimpleSchemaTreeVisitor<>(false) + { + @Override + public Map visitUserSchema(UserSchema schema, Path path, Void v) + { + Map r = Map.of(schema.getSchemaPath(),schema); + return visitAndReduce(schema.getUserSchemas(false), path, null, r); + } + + @Override + public Map reduce(Map r1, Map r2) + { + if (null == r1 || null == r2) + return null==r1 && null==r2 ? Map.of() : null==r1 ? r2 : r1; + var ret = new TreeMap(); + ret.putAll(r1); + ret.putAll(r2); + return ret; + } + }; + + // DefaultSchema does not implement UserSchema which is inconvenient. + TreeMap ret = new TreeMap<>(); + for (String name : root.getUserSchemaNames(false)) + { + UserSchema s = root.getUserSchema(name); + if (null != s) + { + var res = visitor.visit(s, null, null); + ret.putAll(res); + } + } + return ret; + } + + public static JSONObject listTablesForSchema(String fullQuotedName) { SchemaKey fullKey; @@ -9120,7 +9160,14 @@ public static JSONObject listTablesForSchema(String fullQuotedName) table.put("type", null==qd ? "TABLE" : "QUERY"); array.put(table); } - return new JSONObject(Map.of("tables", array)); + + var ret = new JSONObject(); + ret.put("schemaName", schema.getName()); + ret.put("fullQuotedName", schema.getSchemaPath().toSQLString()); + if (isNotBlank(schema.getDescription())) + ret.put("description", schema.getDescription()); + ret.put("tables", array); + return ret; } public static JSONObject listColumnsForTable(String fullQuotedName) From 744459facb70741869f6f3ac5711a9965fa351c0 Mon Sep 17 00:00:00 2001 From: labkey-matthewb Date: Tue, 23 Dec 2025 16:54:52 -0800 Subject: [PATCH 07/25] McpService checkpoint --- api/build.gradle | 39 ++ .../org/labkey/api/module/McpProvider.java | 15 + api/src/org/labkey/api/mpc/McpContext.java | 81 ++++ api/src/org/labkey/api/mpc/McpService.java | 55 +++ core/src/org/labkey/core/CoreModule.java | 7 + .../org/labkey/core/mpc/McpServiceImpl.java | 456 ++++++++++++++++++ query/src/org/labkey/query/QueryModule.java | 8 + .../query/controllers/QueryController.java | 174 +++---- .../labkey/query/controllers/QueryMcp.java | 50 ++ 9 files changed, 770 insertions(+), 115 deletions(-) create mode 100644 api/src/org/labkey/api/module/McpProvider.java create mode 100644 api/src/org/labkey/api/mpc/McpContext.java create mode 100644 api/src/org/labkey/api/mpc/McpService.java create mode 100644 core/src/org/labkey/core/mpc/McpServiceImpl.java create mode 100644 query/src/org/labkey/query/controllers/QueryMcp.java diff --git a/api/build.gradle b/api/build.gradle index 3d64f9ce8d6..15b6703d81d 100644 --- a/api/build.gradle +++ b/api/build.gradle @@ -1029,6 +1029,45 @@ dependencies { ) ) + BuildUtils.addExternalDependency( + project, + new ExternalDependency( + "org.springframework.ai:spring-ai-starter-mcp-server-webmvc:${springAiVersion}", + "spring-ai-starter-mcp-server-webmvc", + "spring-ai", + "https://github.com/spring-projects/spring-ai", + ExternalDependency.APACHE_2_LICENSE_NAME, + ExternalDependency.APACHE_2_LICENSE_URL, + "MPC Servlet" + ) + ) + + BuildUtils.addExternalDependency( + project, + new ExternalDependency( + "org.springframework.ai:spring-ai-bom:${springAiVersion}", + "spring-ai-bom", + "spring-ai", + "https://github.com/spring-projects/spring-ai", + ExternalDependency.APACHE_2_LICENSE_NAME, + ExternalDependency.APACHE_2_LICENSE_URL, + "MPC Servlet" + ) + ) + + BuildUtils.addExternalDependency( + project, + new ExternalDependency( + "com.google.genai:google-genai:1.15.0", + "GENAI", + "GENAI", + "https://google.com/", + "???", + "???", + "GenAI" + ) + ) + jspImplementation files(project.tasks.jar) jspImplementation apache, jackson, spring } diff --git a/api/src/org/labkey/api/module/McpProvider.java b/api/src/org/labkey/api/module/McpProvider.java new file mode 100644 index 00000000000..1e42582ab53 --- /dev/null +++ b/api/src/org/labkey/api/module/McpProvider.java @@ -0,0 +1,15 @@ +package org.labkey.api.module; + +import io.modelcontextprotocol.server.McpServerFeatures; +import org.springframework.ai.tool.ToolCallback; + +import java.util.List; + +public interface McpProvider +{ + List getMcpTools(); + + List getMcpPrompts(); + + List getMcpResources(); +} diff --git a/api/src/org/labkey/api/mpc/McpContext.java b/api/src/org/labkey/api/mpc/McpContext.java new file mode 100644 index 00000000000..dcae3cfb484 --- /dev/null +++ b/api/src/org/labkey/api/mpc/McpContext.java @@ -0,0 +1,81 @@ +package org.labkey.api.mpc; + +import org.jetbrains.annotations.NotNull; +import org.labkey.api.data.Container; +import org.labkey.api.security.User; +import org.labkey.api.security.permissions.ReadPermission; +import org.labkey.api.view.UnauthorizedException; +import org.labkey.api.writer.ContainerUser; +import org.springframework.ai.chat.model.ToolContext; +import java.util.Map; + +public class McpContext implements ContainerUser +{ + final User user; + final Container container; + + private McpContext(ContainerUser ctx) + { + this.container = ctx.getContainer(); + this.user = ctx.getUser(); + } + + McpContext(Container container, User user) + { + if (!container.hasPermission(user, ReadPermission.class)) + throw new UnauthorizedException(); + this.container = container; + this.user = user; + } + + ToolContext getToolContext() + { + return new ToolContext(Map.of("container", getContainer(), "user", getUser())); + } + + + @Override + public Container getContainer() + { + return container; + } + + @Override + public User getUser() + { + return user; + } + + + // + // I'd like to get away from using ThreadLocal, but I haven't + // researched if there are other ways to pass context around to Tools registerd by McpService + // + + private static final ThreadLocal contexts = new ThreadLocal(); + + public static @NotNull McpContext get() + { + var ret = contexts.get(); + if (null == ret) + throw new IllegalStateException("McpContext is not set"); + return ret; + } + + public static AutoCloseable withContext(ContainerUser ctx) + { + return with(new McpContext(ctx)); + } + + public static AutoCloseable withContext(Container container, User user) + { + return with(new McpContext(container, user)); + } + + private static AutoCloseable with(McpContext ctx) + { + final McpContext prev = contexts.get(); + contexts.set(ctx); + return () -> contexts.set(prev); + } +} diff --git a/api/src/org/labkey/api/mpc/McpService.java b/api/src/org/labkey/api/mpc/McpService.java new file mode 100644 index 00000000000..be976579d81 --- /dev/null +++ b/api/src/org/labkey/api/mpc/McpService.java @@ -0,0 +1,55 @@ +package org.labkey.api.mpc; + + +import com.google.genai.Chat; +import io.modelcontextprotocol.server.McpServerFeatures; +import jakarta.servlet.http.HttpSession; +import org.jetbrains.annotations.NotNull; +import org.labkey.api.module.McpProvider; +import org.labkey.api.services.ServiceRegistry; +import org.springframework.ai.tool.ToolCallback; + +import java.util.List; + + +public interface McpService +{ + static McpService get() + { + return ServiceRegistry.get().getService(McpService.class); + } + + static void setInstance(McpService service) + { + ServiceRegistry.get().registerService(McpService.class, service); + } + + default void register(McpProvider mcp) + { + registerTools(mcp.getMcpTools()); + registerPrompts(mcp.getMcpPrompts()); + registerResources(mcp.getMcpResources()); + } + + void registerTools(@NotNull List tools); + + void registerPrompts(@NotNull List prompts); + + void registerResources(@NotNull List resources); + + @NotNull List listTools(); + + @NotNull List listPrompts(); + + @NotNull List listResources(); + + + /* */ + // This probably belongs in its own LLM Service, but it's here for prototyping at the moment + // This is hard-coded to use Gemini (switch to using Spring-AI wrapper or maybe LangChain4j?) + // For now there is no more than one chat session per session! The caller must keep track of prompts sent. + + Chat getChat(HttpSession session); + String sendMessage(Chat chat, String message); + /* */ +} diff --git a/core/src/org/labkey/core/CoreModule.java b/core/src/org/labkey/core/CoreModule.java index ee5d38b3ba9..6cfbf9ad4d7 100644 --- a/core/src/org/labkey/core/CoreModule.java +++ b/core/src/org/labkey/core/CoreModule.java @@ -97,6 +97,7 @@ import org.labkey.api.module.SchemaUpdateType; import org.labkey.api.module.SpringModule; import org.labkey.api.module.Summary; +import org.labkey.api.mpc.McpService; import org.labkey.api.notification.EmailMessage; import org.labkey.api.notification.EmailService; import org.labkey.api.notification.NotificationMenuView; @@ -252,6 +253,7 @@ import org.labkey.core.login.LoginController; import org.labkey.core.metrics.SimpleMetricsServiceImpl; import org.labkey.core.metrics.WebSocketConnectionManager; +import org.labkey.core.mpc.McpServiceImpl; import org.labkey.core.notification.EmailPreferenceConfigServiceImpl; import org.labkey.core.notification.EmailPreferenceContainerListener; import org.labkey.core.notification.EmailPreferenceUserListener; @@ -1280,6 +1282,8 @@ public void moduleStartupComplete(ServletContext servletContext) CoreMigrationSchemaHandler.register(); Encryption.checkMigration(); + + McpServiceImpl.get().startMpcServer(); } // Issue 7527: Auto-detect missing SQL views and attempt to recreate @@ -1308,6 +1312,9 @@ public void registerServlets(ServletContext servletCtx) _webdavServletDynamic = servletCtx.addServlet("static", new WebdavServlet(true)); _webdavServletDynamic.setMultipartConfig(SpringActionController.getMultiPartConfigElement()); _webdavServletDynamic.addMapping("/_webdav/*"); + + McpService.setInstance(new McpServiceImpl()); + McpServiceImpl.get().registerServlets(servletCtx); } @Override diff --git a/core/src/org/labkey/core/mpc/McpServiceImpl.java b/core/src/org/labkey/core/mpc/McpServiceImpl.java new file mode 100644 index 00000000000..99a021ad1b0 --- /dev/null +++ b/core/src/org/labkey/core/mpc/McpServiceImpl.java @@ -0,0 +1,456 @@ +package org.labkey.core.mpc; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.genai.Chat; +import com.google.genai.Client; +import com.google.genai.types.FunctionCall; +import com.google.genai.types.FunctionDeclaration; +import com.google.genai.types.GenerateContentConfig; +import com.google.genai.types.GenerateContentResponse; +import com.google.genai.types.Schema; +import io.modelcontextprotocol.json.McpJsonMapper; +import io.modelcontextprotocol.server.McpServer; +import io.modelcontextprotocol.server.McpServerFeatures; +import io.modelcontextprotocol.server.McpSyncServer; +import io.modelcontextprotocol.server.transport.HttpServletStreamableServerTransportProvider; +import io.modelcontextprotocol.spec.McpSchema; +import jakarta.servlet.ServletContext; +import jakarta.servlet.ServletException; +import jakarta.servlet.ServletRequest; +import jakarta.servlet.ServletResponse; +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletRequestWrapper; +import jakarta.servlet.http.HttpServletResponse; +import jakarta.servlet.http.HttpSession; +import org.apache.commons.lang3.StringUtils; +import org.jetbrains.annotations.NotNull; +import org.json.JSONObject; +import org.labkey.api.collections.CopyOnWriteHashMap; +import org.labkey.api.mpc.McpService; +import org.labkey.api.util.ContextListener; +import org.labkey.api.util.JsonUtil; +import org.labkey.api.util.SessionHelper; +import org.labkey.api.util.ShutdownListener; +import org.springframework.ai.mcp.McpToolUtils; +import org.springframework.ai.support.ToolCallbacks; +import org.springframework.ai.tool.ToolCallback; +import org.springframework.ai.tool.annotation.Tool; +import org.springframework.mock.web.MockHttpServletRequest; +import org.springframework.mock.web.MockHttpServletResponse; +import reactor.core.publisher.Mono; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import static org.apache.commons.lang3.StringUtils.isBlank; + + +public class McpServiceImpl implements McpService +{ + public static final String MESSAGE_ENDPOINT = "/_mcp/message"; + public static final String SSE_ENDPOINT = "/_mcp/sse"; + + private final CopyOnWriteHashMap toolMap = new CopyOnWriteHashMap<>(); + private final CopyOnWriteHashMap promptMap = new CopyOnWriteHashMap<>(); + private final CopyOnWriteHashMap resourceMap = new CopyOnWriteHashMap<>(); + + private final ObjectMapper objectMapper = JsonUtil.DEFAULT_MAPPER; + private final _McpServlet mcpServlet = new _McpServlet(JsonUtil.DEFAULT_MAPPER, MESSAGE_ENDPOINT, SSE_ENDPOINT); + + + public static McpServiceImpl get() + { + return (McpServiceImpl) McpService.get(); + } + + + /** + * Called by CoreModule.registerServlets() + * The servlet will return SC_SERVICE_UNAVAILABLE until startMcpServer() is called + */ + public void registerServlets(ServletContext servletCtx) + { + var mcpServletDynamic = servletCtx.addServlet("mcpServlet", mcpServlet); + mcpServletDynamic.setAsyncSupported(true); + mcpServletDynamic.addMapping(MESSAGE_ENDPOINT + "/*"); + mcpServletDynamic.addMapping(SSE_ENDPOINT + "/*"); + } + + + public static class HelloWorld + { + @Tool(description = "Call this tool when starting a new conversation") + String hello() + { + return "hello world!"; + } + + @Tool(description = "Call this tool when ending a conversation") + String bye() + { + return "bye now"; + } + } + + + static McpServerFeatures.SyncToolSpecification syncTool(ToolCallback tool) + { + return McpToolUtils.toSyncToolSpecification(tool, null); + } + + + public void startMpcServer() + { + mcpServlet.startMcpServer(); + } + + + @Override + public void registerTools(@NotNull List tools) + { + tools.forEach(tool -> toolMap.put(tool.getToolDefinition().name(), tool)); + } + + @Override + public void registerPrompts(@NotNull List prompts) + { + prompts.forEach(prompt -> promptMap.put(prompt.prompt().name(), prompt)); + } + + @Override + public void registerResources(@NotNull List resources) + { + resources.forEach(resource -> resourceMap.put(resource.resource().name(), resource)); + } + + + @Override + public @NotNull List listTools() + { + return new ArrayList<>(toolMap.values()); + } + + public List tools() + { + McpJsonMapper mapper = McpJsonMapper.getDefault(); + return toolMap.values().stream().map(ToolCallback::getToolDefinition).map(td -> + McpSchema.Tool.builder() + .name(td.name()) + .description(td.description()) + .inputSchema(mapper, td.inputSchema()) + .build() + ).toList(); + } + + @Override + public @NotNull List listPrompts() + { + return List.of(); + } + + + @Override + public @NotNull List listResources() + { + return List.of(); + } + + + private class _McpServlet extends HttpServlet // wraps HttpServletSseServerTransportProvider + { + HttpServletStreamableServerTransportProvider transportProvider = null; + McpSyncServer mcpServer = null; + + _McpServlet(ObjectMapper objectMapper, String messageEndpoint, String sseEndpoint) + { +// transportProvider = HttpServletSseServerTransportProvider.builder() +// .jsonMapper(McpJsonMapper.getDefault()) +// .messageEndpoint(messageEndpoint) +// .sseEndpoint(sseEndpoint) +// .build(); + + transportProvider = HttpServletStreamableServerTransportProvider.builder() + .jsonMapper(McpJsonMapper.getDefault()) + .mcpEndpoint(messageEndpoint) + .build(); + } + + void startMcpServer() + { + ToolCallback[] toolCallbacks = ToolCallbacks.from(new HelloWorld()); + var tools = Arrays.stream(toolCallbacks).map(McpToolUtils::toSyncToolSpecification).toList(); + + mcpServer = McpServer.sync(transportProvider) + .tools(tools) +// .capabilities(new McpSchema.ServerCapabilities()) + .build(); + ContextListener.addShutdownListener(new _ShutdownListener()); + } + + @Override + public void service(ServletRequest sreq, ServletResponse sres) throws ServletException, IOException + { + if (!(sreq instanceof HttpServletRequest req) || !(sres instanceof HttpServletResponse res)) + { + // how to set error??? + throw new ServletException("non-HTTP request"); + } + + if (null == mcpServer) + { + res.sendError(HttpServletResponse.SC_SERVICE_UNAVAILABLE); + return; + } + + + if ("POST".equals(req.getMethod())) + { + if (null == req.getParameter("sessionId") && null == req.getSession(true).getAttribute("McpServiceImpl#mcpSessionId")) + { + // USE SSE endpoint to get a sessionId + MockHttpServletRequest mockRequest = new MockHttpServletRequest(req.getServletContext(), "GET", SSE_ENDPOINT); + mockRequest.setAsyncSupported(true); + MockHttpServletResponse mockResponse = new MockHttpServletResponse(); + transportProvider.service(mockRequest, mockResponse); + String body = new String(mockResponse.getContentAsByteArray(), StandardCharsets.UTF_8); + String mcpSessionId = StringUtils.substringBetween(body, "sessionId=", "\n"); + req.getSession(true).setAttribute("McpServiceImpl#mcpSessionId", mcpSessionId); + mockRequest.close(); + mockResponse.getOutputStream().close(); + } + + req = new HttpServletRequestWrapper(req) + { + @Override + public String getParameter(String name) + { + var ret = super.getParameter(name); + if (null == ret && "sessionId".equals(name)) + return String.valueOf(Objects.requireNonNull(((HttpServletRequest)getRequest()).getSession(true).getAttribute("McpServiceImpl#mcpSessionId"))); + return ret; + } + }; + } + transportProvider.service(req, res); + } + + public Mono closeGracefully() + { + if (null != transportProvider) + return transportProvider.closeGracefully(); + return Mono.empty(); + } + + /* + @Override + public Mono closeGracefully() + { + return super.closeGracefully(); + } + + @Override + public void destroy() + { + super.destroy(); + } + + @Override + protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException + { + if (!initialized) + { + response.sendError(HttpServletResponse.SC_SERVICE_UNAVAILABLE); + return; + } + super.doGet(request, response); + } + + @Override + protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException + { + if (!initialized) + { + response.sendError(HttpServletResponse.SC_SERVICE_UNAVAILABLE); + return; + } + + // spring ai requires call to SSE first to get a sessionId???? + if (null == request.getParameter("sessionId")) + { + MockHttpServletRequest mockRequest = new MockHttpServletRequest(request.getServletContext(), "GET", request.getRequestURI()); + MockHttpServletResponse mockResponse = new MockHttpServletResponse(); + doGet(mockRequest, mockResponse); + String body = new String(mockResponse.getContentAsByteArray(), StandardCharsets.UTF_8); + String sessionId = StringUtils.substringBetween(body, "sessionId\":\"", "\""); + request.setAttribute("sessionId", sessionId); + request = new HttpServletRequestWrapper(request) + { + @Override + public String getParameter(String name) + { + if ("sessionId".equals(name)) + return sessionId; + return super.getParameter(name); + } + }; + } + + super.doPost(request, response); + } + + @Override + public Mono notifyClients(String method, Object params) + { + return super.notifyClients(method, params); + } + + @Override + public void setSessionFactory(McpServerSession.Factory sessionFactory) + { + super.setSessionFactory(sessionFactory); + initialized = true; + } + + @Override + protected void service(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException + { + super.service(req, resp); + } + + @Override + public void service(ServletRequest req, ServletResponse res) throws ServletException, IOException + { + super.service(req, res); + } + */ + } + + + // ShutdownListener + + class _ShutdownListener implements ShutdownListener + { + @Override + public String getName() + { + return "MpcService"; + } + + + Mono closing = null; + + @Override + public void shutdownPre() + { + closing = mcpServlet.closeGracefully(); + } + + @Override + public void shutdownStarted() + { + if (null == closing) + closing = mcpServlet.closeGracefully(); + closing.block(Duration.ofSeconds(5)); + } + } + + + + /* GEMINI CHAT SERVICE */ + + + String getModel() + { + return "gemini-2.5-flash"; + } + + + @Override + public Chat getChat(HttpSession session) + { + return SessionHelper.getAttribute(session, Chat.class.getName(), () -> { + Client client = new Client(); + + List fns = new ArrayList<>(); + for (var tc : listTools()) + { + var inputSchema = Schema.fromJson(tc.getToolDefinition().inputSchema()); + var fd = FunctionDeclaration.builder() + .name(tc.getToolDefinition().name()) + .description(tc.getToolDefinition().description()) + .parameters(inputSchema); + fns.add(fd.build()); + } + + GenerateContentConfig config = GenerateContentConfig.builder() + .tools( com.google.genai.types.Tool.builder().functionDeclarations(fns)) + .build(); + Chat chatSession = client.chats.create(getModel(), config); + + return chatSession; + }); + } + + public String sendMessage(Chat chatSession, String message) + { + org.labkey.api.mpc.McpContext.get(); + + GenerateContentResponse response; + List functionCalls; + int sends = 0; + + response = chatSession.sendMessage(message); + sends = sends + 1; + functionCalls = response.functionCalls(); + + while (sends < 3 && null != functionCalls && !functionCalls.isEmpty()) + { + StringBuilder sb = new StringBuilder(); + for (var call : functionCalls) + { + if (call.name().isEmpty()) + break; + var tool = toolMap.get(call.name().get()); + if (null == tool) // ERROR? + continue; + var argsMap = call.args().isEmpty() ? Map.of() : call.args().get(); + var argsString = new JSONObject(argsMap); + String result = tool.call(argsString.toString(), null); + // TODO add context about call and parameters to response? + sb.append(result).append("\n\n"); + } + response = chatSession.sendMessage(sb.toString()); + functionCalls = response.functionCalls(); + } + + // if text is empty and sends > 1 retry the original prompt + var ret = response.text(); + if (isBlank(ret) && sends > 1) + { + response = chatSession.sendMessage(message); + ret = response.text(); + if (isBlank(ret)) + ret = "Too many tool calls. Try again."; + } + return ret; + } + + +// public GenerateContentResponse sendPrompt(Chat chatSession, String promptName) +// { +// // is McpContext set? +// McpContext.get(); +// +// var p = Objects.requireNonNull(promptMap.get(promptName)); +// p.promptHandler(). +// +// return sendMesssage(chatSession); +// } +} diff --git a/query/src/org/labkey/query/QueryModule.java b/query/src/org/labkey/query/QueryModule.java index 7485cfe98ca..cf245a7d05e 100644 --- a/query/src/org/labkey/query/QueryModule.java +++ b/query/src/org/labkey/query/QueryModule.java @@ -35,6 +35,7 @@ import org.labkey.api.module.DefaultModule; import org.labkey.api.module.Module; import org.labkey.api.module.ModuleContext; +import org.labkey.api.mpc.McpService; import org.labkey.api.pipeline.PipelineService; import org.labkey.api.query.DefaultSchema; import org.labkey.api.query.JavaExportScriptFactory; @@ -93,6 +94,7 @@ import org.labkey.query.audit.QueryUpdateAuditProvider; import org.labkey.query.controllers.OlapController; import org.labkey.query.controllers.QueryController; +import org.labkey.query.controllers.QueryMcp; import org.labkey.query.controllers.SqlController; import org.labkey.query.jdbc.QueryDriver; import org.labkey.query.olap.MemberSet; @@ -237,6 +239,12 @@ public QuerySchema createSchema(DefaultSchema schema, Module module) "Allow for lookup fields in product folders to query across all folders within the top-level folder.", false); OptionalFeatureService.get().addExperimentalFeatureFlag(QueryServiceImpl.EXPERIMENTAL_PRODUCT_PROJECT_DATA_LISTING_SCOPED, "Product folders display folder-specific data", "Only list folder-specific data within product folders.", false); + + var mcp = McpService.get(); + if (null != mcp) + { + mcp.register(new QueryMcp()); + } } diff --git a/query/src/org/labkey/query/controllers/QueryController.java b/query/src/org/labkey/query/controllers/QueryController.java index 0d883e782c3..c99126f3b77 100644 --- a/query/src/org/labkey/query/controllers/QueryController.java +++ b/query/src/org/labkey/query/controllers/QueryController.java @@ -19,13 +19,11 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.genai.Chat; +import com.google.genai.errors.ClientException; import com.google.genai.errors.ServerException; -import com.google.genai.types.FunctionDeclaration; -import com.google.genai.types.GenerateContentConfig; +import com.google.genai.types.Content; import com.google.genai.types.Part; -import com.google.genai.types.Schema; -import com.google.genai.types.Tool; -import com.google.genai.types.Type; import jakarta.servlet.ServletException; import jakarta.servlet.http.HttpServletRequest; import jakarta.servlet.http.HttpServletResponse; @@ -165,6 +163,7 @@ import org.labkey.api.gwt.client.model.GWTPropertyDescriptor; import org.labkey.api.module.ModuleHtmlView; import org.labkey.api.module.ModuleLoader; +import org.labkey.api.mpc.McpService; import org.labkey.api.pipeline.RecordedAction; import org.labkey.api.query.AbstractQueryImportAction; import org.labkey.api.query.AbstractQueryUpdateService; @@ -330,7 +329,6 @@ import java.io.IOException; import java.io.OutputStream; import java.io.PrintWriter; -import java.lang.reflect.Method; import java.nio.file.Path; import java.sql.Connection; import java.sql.ResultSet; @@ -355,19 +353,12 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import com.google.common.collect.ImmutableList; -import com.google.genai.Chat; -import com.google.genai.Client; -import com.google.genai.types.Content; -import com.google.genai.types.GenerateContentResponse; - import static org.apache.commons.lang3.StringUtils.isBlank; import static org.apache.commons.lang3.StringUtils.isNotBlank; import static org.apache.commons.lang3.StringUtils.trimToEmpty; import static org.labkey.api.action.ApiJsonWriter.CONTENT_TYPE_JSON; import static org.labkey.api.assay.AssayFileWriter.ensureUploadDirectory; import static org.labkey.api.data.DbScope.NO_OP_TRANSACTION; -import static org.labkey.api.data.views.DataViewProvider.EditInfo.Property.name; import static org.labkey.api.query.AbstractQueryUpdateService.saveFile; import static org.labkey.api.util.DOM.BR; import static org.labkey.api.util.DOM.DIV; @@ -8932,35 +8923,11 @@ String getModel() Chat getChat(String currentSchema) { - return SessionHelper.getAttribute(getViewContext().getRequest(), Chat.class.getName(), () -> { - Client client = new Client(); - - Schema columnsMetaDataParameters = Schema.builder() - .type(Type.Known.OBJECT) - .properties(Map.of("quotedTableName", Schema.builder() - .type(Type.Known.STRING) - .description("Fully qualified table name as it would appear in SQL e.g. \"schema\".\"table\"") - .build())) - .required("quotedTableName") - .build(); - var listColumnMetaDataFn = FunctionDeclaration.builder().name("listColumnsForTable").description("Provide column metadata for a sql table. This tool Will also return SQL source for saved queries.").parameters(columnsMetaDataParameters); - - Schema tablesMetaDataParameters = Schema.builder() - .type(Type.Known.OBJECT) - .properties(Map.of("quotedSchemaName", Schema.builder() - .type(Type.Known.STRING) - .description("Fully qualified schema name as it would appear in SQL e.g. \"schema\"") - .build())) - .required("quotedSchemaName") - .build(); - var listTablesMetaDataFn = FunctionDeclaration.builder().name("listTablesForSchema").description("Provide column metadata for a database table.").parameters(tablesMetaDataParameters); - - var tools = Tool.builder().functionDeclarations(listColumnMetaDataFn, listTablesMetaDataFn); - - GenerateContentConfig config = GenerateContentConfig.builder().tools(tools).build(); - - Chat chatSession = client.chats.create(getModel(), config); + HttpSession session = getViewContext().getRequest().getSession(true); + Chat chatSession = McpService.get().getChat(session); + if (Boolean.FALSE == SessionHelper.getAttribute(session, "QueryController#queryChatInitialized", Boolean.FALSE)) + { StringBuilder serviceMessage = new StringBuilder(); serviceMessage.append("Your job is to generate SQL statements. Here is some reference material formatted as markdown:\n").append(getSQLHelp()).append("\n"); serviceMessage.append("NOTE: please prefer using lookup syntax rather than JOIN where possible.\n"); @@ -8987,23 +8954,24 @@ Chat getChat(String currentSchema) } } - chatSession.sendMessage(serviceMessage.toString()); - return chatSession; - }); + McpService.get().sendMessage(chatSession, serviceMessage.toString()); + SessionHelper.getAttribute(session, "QueryController#queryChatInitialized", Boolean.TRUE); + } + return chatSession; } @Override public Object execute(PromptForm form, BindException errors) throws Exception { - Chat chatSession = getChat(form.getSchemaName()); - - String prompt = form.getPrompt(); - for (int retry=0 ; retry < 5 ; retry++) + try (var mcpPush = org.labkey.api.mpc.McpContext.withContext(getViewContext())) { - GenerateContentResponse response; + Chat chatSession = getChat(form.getSchemaName()); + String prompt = form.getPrompt(); + String responseText; + try { - response = chatSession.sendMessage(prompt); + responseText = McpService.get().sendMessage(chatSession, prompt); } catch (ServerException x) { @@ -9013,82 +8981,58 @@ public Object execute(PromptForm form, BindException errors) throws Exception "text", "ERROR: " + x.getMessage(), "success", Boolean.FALSE)); } - var functionCalls = response.functionCalls(); - if (null == functionCalls || functionCalls.isEmpty()) - { - var sql = extractSql(response.text()); - var text = null==sql ? response.text() : null; - /* VALIDATE SQL */ - if (null != sql) + var sql = extractSql(responseText); + var text = null == sql ? responseText : null; + + /* VALIDATE SQL */ + if (null != sql) + { + QuerySchema schema = DefaultSchema.get(getUser(), getContainer()).getSchema("study"); + try { - QuerySchema schema = DefaultSchema.get(getUser(), getContainer()).getSchema("study"); - try - { - TableInfo ti = QueryService.get().createTable(schema, sql, null, true); - var warnings = ti.getWarnings(); - if (null != warnings) - { - var warning = warnings.stream().findFirst(); - if (warning.isPresent()) - throw warning.get(); - } - } - catch (QueryException x) + TableInfo ti = QueryService.get().createTable(schema, sql, null, true); + var warnings = ti.getWarnings(); + if (null != warnings) { - String validationPrompt = "That SQL caused the " + (x instanceof QueryParseWarning ? "warning" : "error") + " below, can you attempt to fix this?\n```" + x.getMessage() + "```"; - response = chatSession.sendMessage(validationPrompt); - text = response.text(); - var newSQL = extractSql(response.text()); - if (isNotBlank(newSQL)) - sql = newSQL; + var warning = warnings.stream().findFirst(); + if (warning.isPresent()) + throw warning.get(); } } - - System.err.println(chatSession.getHistory(true)); - var ret = new JSONObject(Map.of( - "model", getModel(), - "success", Boolean.TRUE)); - if (null != sql) - ret.put("sql",sql); - if (null != text) - ret.put("text", text); - return ret; - } - StringBuilder fnPrompt = new StringBuilder(); - for (var functionCall : response.functionCalls()) - { - var functionName = functionCall.name().orElse(null); - if ("listColumnsForTable".equals(functionName)) - { - var quotedName = String.valueOf(functionCall.args().get().get("quotedTableName")); - var res = listColumnsForTable(quotedName); - fnPrompt.append("Here is additional metadata for table " + quotedName + " formatted as JSON:\n```").append(res).append("```\n\n"); - } - else if ("listTablesForSchema".equals(functionName)) + catch (QueryException x) { - var quotedName = String.valueOf(functionCall.args().get().get("quotedSchemaName")); - var res = listTablesForSchema(quotedName); - fnPrompt.append("Here is additional metadata for schema " + quotedName + " formatted as JSON:\n```").append(res).append("```\n\n"); + String validationPrompt = "That SQL caused the " + (x instanceof QueryParseWarning ? "warning" : "error") + " below, can you attempt to fix this?\n```" + x.getMessage() + "```"; + responseText = McpService.get().sendMessage(chatSession, validationPrompt); + var newSQL = extractSql(responseText); + if (isNotBlank(newSQL)) + sql = newSQL; + text = responseText; } } - prompt = fnPrompt.toString(); - } - // FALLBACK??? - GenerateContentResponse response = chatSession.sendMessage(form.getPrompt()); - System.err.println(chatSession.getHistory(true)); - var ret = new JSONObject(Map.of( - "text", response.text(), - "user", getViewContext().getUser().getName(), - "model", getModel(), - "success", Boolean.TRUE)); - var functionCalls = response.functionCalls(); - if (null != functionCalls && !functionCalls.isEmpty()) - ret.put("functionCall", functionCalls.get(0).toString()); - return ret; + System.err.println(chatSession.getHistory(true)); + var ret = new JSONObject(Map.of( + "model", getModel(), + "success", Boolean.TRUE)); + if (null != sql) + ret.put("sql", sql); + if (null != text) + ret.put("text", text); + return ret; + } + catch (ClientException ex) + { + var ret = new JSONObject(Map.of( + "text", ex.getMessage(), + "user", getViewContext().getUser().getName(), + "model", getModel(), + "success", Boolean.FALSE)); + return ret; + } } + String extractSql(String text) { if (text.startsWith("SELECT ")) diff --git a/query/src/org/labkey/query/controllers/QueryMcp.java b/query/src/org/labkey/query/controllers/QueryMcp.java new file mode 100644 index 00000000000..305f17c65ea --- /dev/null +++ b/query/src/org/labkey/query/controllers/QueryMcp.java @@ -0,0 +1,50 @@ +package org.labkey.query.controllers; + +import io.modelcontextprotocol.server.McpServerFeatures; +import org.labkey.api.module.McpProvider; +import org.springframework.ai.support.ToolCallbacks; +import org.springframework.ai.tool.ToolCallback; +import org.springframework.ai.tool.annotation.Tool; +import org.springframework.ai.tool.annotation.ToolParam; + +import java.util.Arrays; +import java.util.List; + +public class QueryMcp implements McpProvider +{ + @Override + public List getMcpTools() + { + ToolCallback[] queryTools = ToolCallbacks.from(this); + return Arrays.asList(queryTools); + } + + @Override + public List getMcpPrompts() + { + return List.of(); + } + + @Override + public List getMcpResources() + { + return List.of(); + } + + @Tool(description = "Provide column metadata for a sql table. This tool will also return SQL source for saved queries.") + String listColumnMetaData(@ToolParam(description = "Fully qualified table name as it would appear in SQL e.g. \"schema\".\"table\"") String fullQuotedTableName) + { + var json = QueryController.listColumnsForTable(fullQuotedTableName); + // can I just return a JSONObject + return json.toString(); + } + + @Tool(description = "Provide list of tables within the provided schema.") + String listTablesForSchema(@ToolParam(description = "Fully qualified schema name as it would appear in SQL e.g. \"schema\"") String quotedSchemaName) + { + var json = QueryController.listTablesForSchema(quotedSchemaName); + // can I just return a JSONObject + return json.toString(); + } +} + From 099b3bdf0cfcc032750a5e2ec07f9ce7b92767df Mon Sep 17 00:00:00 2001 From: labkey-matthewb Date: Tue, 30 Dec 2025 17:53:39 -0800 Subject: [PATCH 08/25] AbstractAgentAction migration to spring-ai for chat client --- api/build.gradle | 26 ++++ .../labkey/api/{mpc => mcp}/McpContext.java | 4 +- .../labkey/api/{mpc => mcp}/McpService.java | 8 +- core/src/org/labkey/core/CoreModule.java | 2 +- .../org/labkey/core/mpc/McpServiceImpl.java | 80 +++++++++-- query/src/org/labkey/query/QueryModule.java | 2 +- .../query/controllers/QueryController.java | 124 ++++++++---------- 7 files changed, 160 insertions(+), 86 deletions(-) rename api/src/org/labkey/api/{mpc => mcp}/McpContext.java (96%) rename api/src/org/labkey/api/{mpc => mcp}/McpService.java (85%) diff --git a/api/build.gradle b/api/build.gradle index 15b6703d81d..f0b703a6256 100644 --- a/api/build.gradle +++ b/api/build.gradle @@ -1055,6 +1055,32 @@ dependencies { ) ) + BuildUtils.addExternalDependency( + project, + new ExternalDependency( + "org.springframework.ai:spring-ai-starter-model-google-genai:${springAiVersion}", + "spring-ai-starter-google-genai", + "spring-ai", + "https://github.com/spring-projects/spring-ai", + ExternalDependency.APACHE_2_LICENSE_NAME, + ExternalDependency.APACHE_2_LICENSE_URL, + "LLM Chat Client integration for Gemini" + ) + ) + + BuildUtils.addExternalDependency( + project, + new ExternalDependency( + "org.springframework.ai:spring-ai-client-chat:${springAiVersion}", + "spring-ai-client-chat", + "spring-ai", + "https://github.com/spring-projects/spring-ai", + ExternalDependency.APACHE_2_LICENSE_NAME, + ExternalDependency.APACHE_2_LICENSE_URL, + "LLM Chat Client" + ) + ) + BuildUtils.addExternalDependency( project, new ExternalDependency( diff --git a/api/src/org/labkey/api/mpc/McpContext.java b/api/src/org/labkey/api/mcp/McpContext.java similarity index 96% rename from api/src/org/labkey/api/mpc/McpContext.java rename to api/src/org/labkey/api/mcp/McpContext.java index dcae3cfb484..3f13448d398 100644 --- a/api/src/org/labkey/api/mpc/McpContext.java +++ b/api/src/org/labkey/api/mcp/McpContext.java @@ -1,4 +1,4 @@ -package org.labkey.api.mpc; +package org.labkey.api.mcp; import org.jetbrains.annotations.NotNull; import org.labkey.api.data.Container; @@ -28,7 +28,7 @@ private McpContext(ContainerUser ctx) this.user = user; } - ToolContext getToolContext() + public ToolContext getToolContext() { return new ToolContext(Map.of("container", getContainer(), "user", getUser())); } diff --git a/api/src/org/labkey/api/mpc/McpService.java b/api/src/org/labkey/api/mcp/McpService.java similarity index 85% rename from api/src/org/labkey/api/mpc/McpService.java rename to api/src/org/labkey/api/mcp/McpService.java index be976579d81..2aee92f6b15 100644 --- a/api/src/org/labkey/api/mpc/McpService.java +++ b/api/src/org/labkey/api/mcp/McpService.java @@ -1,4 +1,4 @@ -package org.labkey.api.mpc; +package org.labkey.api.mcp; import com.google.genai.Chat; @@ -7,9 +7,11 @@ import org.jetbrains.annotations.NotNull; import org.labkey.api.module.McpProvider; import org.labkey.api.services.ServiceRegistry; +import org.springframework.ai.chat.client.ChatClient; import org.springframework.ai.tool.ToolCallback; import java.util.List; +import java.util.function.Supplier; public interface McpService @@ -51,5 +53,9 @@ default void register(McpProvider mcp) Chat getChat(HttpSession session); String sendMessage(Chat chat, String message); + + ChatClient getChatSpringAi(HttpSession session, String agentName, Supplier systemPromptSupplier); + String sendMessage(ChatClient chat, String message); + /* */ } diff --git a/core/src/org/labkey/core/CoreModule.java b/core/src/org/labkey/core/CoreModule.java index 6cfbf9ad4d7..f6d93ca7753 100644 --- a/core/src/org/labkey/core/CoreModule.java +++ b/core/src/org/labkey/core/CoreModule.java @@ -97,7 +97,7 @@ import org.labkey.api.module.SchemaUpdateType; import org.labkey.api.module.SpringModule; import org.labkey.api.module.Summary; -import org.labkey.api.mpc.McpService; +import org.labkey.api.mcp.McpService; import org.labkey.api.notification.EmailMessage; import org.labkey.api.notification.EmailService; import org.labkey.api.notification.NotificationMenuView; diff --git a/core/src/org/labkey/core/mpc/McpServiceImpl.java b/core/src/org/labkey/core/mpc/McpServiceImpl.java index 99a021ad1b0..c2d750017ea 100644 --- a/core/src/org/labkey/core/mpc/McpServiceImpl.java +++ b/core/src/org/labkey/core/mpc/McpServiceImpl.java @@ -3,6 +3,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.genai.Chat; import com.google.genai.Client; +import com.google.genai.types.ClientOptions; import com.google.genai.types.FunctionCall; import com.google.genai.types.FunctionDeclaration; import com.google.genai.types.GenerateContentConfig; @@ -27,11 +28,21 @@ import org.jetbrains.annotations.NotNull; import org.json.JSONObject; import org.labkey.api.collections.CopyOnWriteHashMap; -import org.labkey.api.mpc.McpService; +import org.labkey.api.mcp.McpContext; +import org.labkey.api.mcp.McpService; import org.labkey.api.util.ContextListener; import org.labkey.api.util.JsonUtil; import org.labkey.api.util.SessionHelper; import org.labkey.api.util.ShutdownListener; +import org.springframework.ai.chat.client.ChatClient; +import org.springframework.ai.chat.client.advisor.MessageChatMemoryAdvisor; +import org.springframework.ai.chat.memory.ChatMemory; +import org.springframework.ai.chat.memory.ChatMemoryRepository; +import org.springframework.ai.chat.memory.InMemoryChatMemoryRepository; +import org.springframework.ai.chat.memory.MessageWindowChatMemory; +import org.springframework.ai.chat.model.ChatModel; +import org.springframework.ai.google.genai.GoogleGenAiChatModel; +import org.springframework.ai.google.genai.GoogleGenAiChatOptions; import org.springframework.ai.mcp.McpToolUtils; import org.springframework.ai.support.ToolCallbacks; import org.springframework.ai.tool.ToolCallback; @@ -48,6 +59,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.function.Supplier; import static org.apache.commons.lang3.StringUtils.isBlank; @@ -63,6 +75,7 @@ public class McpServiceImpl implements McpService private final ObjectMapper objectMapper = JsonUtil.DEFAULT_MAPPER; private final _McpServlet mcpServlet = new _McpServlet(JsonUtil.DEFAULT_MAPPER, MESSAGE_ENDPOINT, SSE_ENDPOINT); + private final ChatMemoryRepository chatMemoryRepository = new InMemoryChatMemoryRepository(); public static McpServiceImpl get() @@ -369,6 +382,8 @@ public void shutdownStarted() String getModel() { return "gemini-2.5-flash"; +// gemini-2.5-flash-lite is cheaper but it seems to be much worse at SQL than gemini-2.5-flash +// return "gemini-2.5-flash-lite"; } @@ -398,9 +413,10 @@ public Chat getChat(HttpSession session) }); } + @Override public String sendMessage(Chat chatSession, String message) { - org.labkey.api.mpc.McpContext.get(); + // TODO tool context? org.labkey.api.mpc.McpContext.get(); GenerateContentResponse response; List functionCalls; @@ -443,14 +459,54 @@ public String sendMessage(Chat chatSession, String message) } -// public GenerateContentResponse sendPrompt(Chat chatSession, String promptName) -// { -// // is McpContext set? -// McpContext.get(); -// -// var p = Objects.requireNonNull(promptMap.get(promptName)); -// p.promptHandler(). -// -// return sendMesssage(chatSession); -// } + // SPRING AI CHAT SERVICE + @Override + public ChatClient getChatSpringAi(HttpSession session, String agentName, Supplier systemPromptSupplier) + { + return SessionHelper.getAttribute(session, ChatClient.class.getName() + "#" + agentName, () -> + { + String systemPrompt = systemPromptSupplier.get(); + String conversationId = session.getId() + ":" + agentName; + + ClientOptions clientOptions = ClientOptions.builder() + .build(); + + Client genAiClient = Client.builder() + .clientOptions(clientOptions) + .build(); + + GoogleGenAiChatOptions chatOptions = GoogleGenAiChatOptions.builder() + .model(getModel()) + .toolCallbacks(listTools()) + .build(); + ChatModel chatModel = GoogleGenAiChatModel.builder() + .genAiClient(genAiClient) + .defaultOptions(chatOptions) + .build(); + ChatMemory chatMemory = MessageWindowChatMemory.builder() + .maxMessages(100) + .chatMemoryRepository(chatMemoryRepository) + .build(); + return ChatClient.builder(chatModel) + .defaultOptions(chatOptions) + .defaultAdvisors(MessageChatMemoryAdvisor.builder(chatMemory) + .conversationId(conversationId) + .build()) + .defaultSystem(systemPrompt) + .build(); + }); + } + + + @Override + public String sendMessage(ChatClient chatSession, String message) + { + String content; + content = chatSession + .prompt(message) + .toolContext(McpContext.get().getToolContext().getContext()) + .call() + .content(); + return content; + } } diff --git a/query/src/org/labkey/query/QueryModule.java b/query/src/org/labkey/query/QueryModule.java index cf245a7d05e..58fd212f4af 100644 --- a/query/src/org/labkey/query/QueryModule.java +++ b/query/src/org/labkey/query/QueryModule.java @@ -35,7 +35,7 @@ import org.labkey.api.module.DefaultModule; import org.labkey.api.module.Module; import org.labkey.api.module.ModuleContext; -import org.labkey.api.mpc.McpService; +import org.labkey.api.mcp.McpService; import org.labkey.api.pipeline.PipelineService; import org.labkey.api.query.DefaultSchema; import org.labkey.api.query.JavaExportScriptFactory; diff --git a/query/src/org/labkey/query/controllers/QueryController.java b/query/src/org/labkey/query/controllers/QueryController.java index bd9790073bc..6207aa43863 100644 --- a/query/src/org/labkey/query/controllers/QueryController.java +++ b/query/src/org/labkey/query/controllers/QueryController.java @@ -22,8 +22,6 @@ import com.google.genai.Chat; import com.google.genai.errors.ClientException; import com.google.genai.errors.ServerException; -import com.google.genai.types.Content; -import com.google.genai.types.Part; import jakarta.servlet.ServletException; import jakarta.servlet.http.HttpServletRequest; import jakarta.servlet.http.HttpServletResponse; @@ -161,9 +159,12 @@ import org.labkey.api.files.FileContentService; import org.labkey.api.gwt.client.AuditBehaviorType; import org.labkey.api.gwt.client.model.GWTPropertyDescriptor; +import org.labkey.api.mcp.McpContext; import org.labkey.api.module.ModuleHtmlView; import org.labkey.api.module.ModuleLoader; -import org.labkey.api.mpc.McpService; +import org.labkey.api.mcp.AbstractAgentAction; +import org.labkey.api.mcp.McpService; +import org.labkey.api.mcp.PromptForm; import org.labkey.api.pipeline.RecordedAction; import org.labkey.api.query.AbstractQueryImportAction; import org.labkey.api.query.AbstractQueryUpdateService; @@ -246,7 +247,6 @@ import org.labkey.api.util.Pair; import org.labkey.api.util.ResponseHelper; import org.labkey.api.util.ReturnURLString; -import org.labkey.api.util.SessionHelper; import org.labkey.api.util.StringExpression; import org.labkey.api.util.StringUtilsLabKey; import org.labkey.api.util.TestContext; @@ -311,6 +311,7 @@ import org.labkey.remoteapi.SelectRowsStreamHack; import org.labkey.remoteapi.query.SelectRowsCommand; import org.labkey.vfs.FileLike; +import org.springframework.ai.chat.client.ChatClient; import org.springframework.beans.MutablePropertyValues; import org.springframework.beans.PropertyValue; import org.springframework.beans.PropertyValues; @@ -8869,10 +8870,9 @@ public void addNavTrail(NavTree root) } - public static class PromptForm + public static class SqlPromptForm extends PromptForm { - String prompt; - String schemaName; + public String schemaName; public String getSchemaName() { @@ -8883,90 +8883,80 @@ public void setSchemaName(String schemaName) { this.schemaName = schemaName; } - - public void setPrompt(String prompt) - { - this.prompt = prompt; - } - - public String getPrompt() - { - return this.prompt; - } } @RequiresPermission(ReadPermission.class) @RequiresLogin - public static class QueryAgentAction extends ReadOnlyApiAction + public static class QueryAgentAction extends AbstractAgentAction { - String getSQLHelp() - { - try - { - return IOUtils.resourceToString("org/labkey/query/controllers/LabKeySql.md", null, QueryController.class.getClassLoader()); - } - catch (IOException x) - { - throw new ConfigurationException("error loading resource", x); - } - } + SqlPromptForm _form; - Content contentFromText(String s) + @Override + public void validateForm(SqlPromptForm sqlPromptForm, Errors errors) { - return Content.fromParts(Part.fromText(s)); + _form = sqlPromptForm; } - String getModel() + @Override + protected String getAgentName() { - return "gemini-2.5-flash"; + return QueryAgentAction.class.getName(); } - Chat getChat(String currentSchema) + @Override + protected String getServicePrompt() { - HttpSession session = getViewContext().getRequest().getSession(true); - Chat chatSession = McpService.get().getChat(session); + StringBuilder serviceMessage = new StringBuilder(); + serviceMessage.append("Your job is to generate SQL statements. Here is some reference material formatted as markdown:\n").append(getSQLHelp()).append("\n"); + serviceMessage.append("NOTE: please prefer using lookup syntax rather than JOIN where possible.\n"); - if (Boolean.FALSE == SessionHelper.getAttribute(session, "QueryController#queryChatInitialized", Boolean.FALSE)) + DefaultSchema defaultSchema = DefaultSchema.get(getUser(), getContainer()); + Map schemaMap = listAllSchemas(defaultSchema); + StringBuilder sb = new StringBuilder(); + for (var schema : schemaMap.values()) { - StringBuilder serviceMessage = new StringBuilder(); - serviceMessage.append("Your job is to generate SQL statements. Here is some reference material formatted as markdown:\n").append(getSQLHelp()).append("\n"); - serviceMessage.append("NOTE: please prefer using lookup syntax rather than JOIN where possible.\n"); - - DefaultSchema defaultSchema = DefaultSchema.get(getUser(), getContainer()); - Map schemaMap = listAllSchemas(defaultSchema); - StringBuilder sb = new StringBuilder(); - for (var schema : schemaMap.values()) - { - sb.append("\t* ").append(schema.getSchemaPath().toSQLString()); - if (isNotBlank(schema.getDescription())) - sb.append("\t").append(schema.getDescription()); - sb.append("\n"); - } - serviceMessage.append("\n\nHere are the available schemas:\n" + sb); + sb.append("\t* ").append(schema.getSchemaPath().toSQLString()); + if (isNotBlank(schema.getDescription())) + sb.append("\t").append(schema.getDescription()); + sb.append("\n"); + } + serviceMessage.append("\n\nHere are the available schemas:\n" + sb); - if (!isBlank(currentSchema)) + if (!isBlank(_form.getSchemaName())) + { + var schema = defaultSchema.getSchema(_form.getSchemaName()); + if (null != schema) { - var schema = defaultSchema.getSchema(currentSchema); - if (null != schema) - { - serviceMessage.append("\n\nCurrent default schema is " + schema.getSchemaPath().toSQLString() + ". This is a list of tables in this schema formatted as JSON\n```" + serviceMessage.append("\n\nCurrent default schema is " + schema.getSchemaPath().toSQLString() + ". This is a list of tables in this schema formatted as JSON\n```" + listColumnsForTable(schema.getSchemaPath().toSQLString()) + "\n```"); - } } + } + return serviceMessage.toString(); + } + - McpService.get().sendMessage(chatSession, serviceMessage.toString()); - SessionHelper.getAttribute(session, "QueryController#queryChatInitialized", Boolean.TRUE); + String getSQLHelp() + { + try + { + return IOUtils.resourceToString("org/labkey/query/controllers/LabKeySql.md", null, QueryController.class.getClassLoader()); + } + catch (IOException x) + { + throw new ConfigurationException("error loading resource", x); } - return chatSession; } @Override - public Object execute(PromptForm form, BindException errors) throws Exception + public Object execute(SqlPromptForm form, BindException errors) throws Exception { - try (var mcpPush = org.labkey.api.mpc.McpContext.withContext(getViewContext())) + // save form here for context in getServicePrompt() + _form = form; + + try (var mcpPush = McpContext.withContext(getViewContext())) { - Chat chatSession = getChat(form.getSchemaName()); + ChatClient chatSession = getChat(); String prompt = form.getPrompt(); String responseText; @@ -8977,7 +8967,6 @@ public Object execute(PromptForm form, BindException errors) throws Exception catch (ServerException x) { return new JSONObject(Map.of( - "model", getModel(), "error", x.getMessage(), "text", "ERROR: " + x.getMessage(), "success", Boolean.FALSE)); @@ -9012,9 +9001,8 @@ public Object execute(PromptForm form, BindException errors) throws Exception } } - System.err.println(chatSession.getHistory(true)); +// System.err.println(chatSession.getHistory(true)); var ret = new JSONObject(Map.of( - "model", getModel(), "success", Boolean.TRUE)); if (null != sql) ret.put("sql", sql); @@ -9027,7 +9015,6 @@ public Object execute(PromptForm form, BindException errors) throws Exception var ret = new JSONObject(Map.of( "text", ex.getMessage(), "user", getViewContext().getUser().getName(), - "model", getModel(), "success", Boolean.FALSE)); return ret; } @@ -9053,7 +9040,6 @@ String extractSql(String text) } } - @RequiresPermission(ReadPermission.class) @RequiresLogin public static class ResetQueryAgentAction extends ReadOnlyApiAction From 78b2fc01e0c4000280a9bf3a78a8a2e5bb803506 Mon Sep 17 00:00:00 2001 From: labkey-matthewb Date: Wed, 31 Dec 2025 17:01:56 -0800 Subject: [PATCH 09/25] migration to spring-ai for chat client test-chat.view (AbstractAgentAction) --- api/src/org/labkey/api/mcp/McpService.java | 15 +- .../org/labkey/core/mpc/McpServiceImpl.java | 200 +++--------------- .../org/labkey/devtools/TestController.java | 36 ++++ .../query/controllers/QueryController.java | 4 +- .../labkey/query/controllers/QueryMcp.java | 37 ++++ .../src/org/labkey/query/view/sourceQuery.jsp | 12 +- 6 files changed, 111 insertions(+), 193 deletions(-) diff --git a/api/src/org/labkey/api/mcp/McpService.java b/api/src/org/labkey/api/mcp/McpService.java index 2aee92f6b15..b7ea667ae89 100644 --- a/api/src/org/labkey/api/mcp/McpService.java +++ b/api/src/org/labkey/api/mcp/McpService.java @@ -1,12 +1,12 @@ package org.labkey.api.mcp; -import com.google.genai.Chat; import io.modelcontextprotocol.server.McpServerFeatures; import jakarta.servlet.http.HttpSession; import org.jetbrains.annotations.NotNull; import org.labkey.api.module.McpProvider; import org.labkey.api.services.ServiceRegistry; +import org.labkey.api.util.HtmlString; import org.springframework.ai.chat.client.ChatClient; import org.springframework.ai.tool.ToolCallback; @@ -45,17 +45,10 @@ default void register(McpProvider mcp) @NotNull List listResources(); + ChatClient getChat(HttpSession session, String agentName, Supplier systemPromptSupplier); - /* */ - // This probably belongs in its own LLM Service, but it's here for prototyping at the moment - // This is hard-coded to use Gemini (switch to using Spring-AI wrapper or maybe LangChain4j?) - // For now there is no more than one chat session per session! The caller must keep track of prompts sent. - - Chat getChat(HttpSession session); - String sendMessage(Chat chat, String message); - - ChatClient getChatSpringAi(HttpSession session, String agentName, Supplier systemPromptSupplier); - String sendMessage(ChatClient chat, String message); + record MessageResponse(String markdown, HtmlString html) {} + MessageResponse sendMessage(ChatClient chat, String message); /* */ } diff --git a/core/src/org/labkey/core/mpc/McpServiceImpl.java b/core/src/org/labkey/core/mpc/McpServiceImpl.java index c2d750017ea..d9ef25b56d4 100644 --- a/core/src/org/labkey/core/mpc/McpServiceImpl.java +++ b/core/src/org/labkey/core/mpc/McpServiceImpl.java @@ -1,14 +1,8 @@ package org.labkey.core.mpc; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.genai.Chat; import com.google.genai.Client; import com.google.genai.types.ClientOptions; -import com.google.genai.types.FunctionCall; -import com.google.genai.types.FunctionDeclaration; -import com.google.genai.types.GenerateContentConfig; -import com.google.genai.types.GenerateContentResponse; -import com.google.genai.types.Schema; import io.modelcontextprotocol.json.McpJsonMapper; import io.modelcontextprotocol.server.McpServer; import io.modelcontextprotocol.server.McpServerFeatures; @@ -26,11 +20,12 @@ import jakarta.servlet.http.HttpSession; import org.apache.commons.lang3.StringUtils; import org.jetbrains.annotations.NotNull; -import org.json.JSONObject; import org.labkey.api.collections.CopyOnWriteHashMap; +import org.labkey.api.markdown.MarkdownService; import org.labkey.api.mcp.McpContext; import org.labkey.api.mcp.McpService; import org.labkey.api.util.ContextListener; +import org.labkey.api.util.HtmlString; import org.labkey.api.util.JsonUtil; import org.labkey.api.util.SessionHelper; import org.labkey.api.util.ShutdownListener; @@ -41,6 +36,7 @@ import org.springframework.ai.chat.memory.InMemoryChatMemoryRepository; import org.springframework.ai.chat.memory.MessageWindowChatMemory; import org.springframework.ai.chat.model.ChatModel; +import org.springframework.ai.chat.model.Generation; import org.springframework.ai.google.genai.GoogleGenAiChatModel; import org.springframework.ai.google.genai.GoogleGenAiChatOptions; import org.springframework.ai.mcp.McpToolUtils; @@ -57,11 +53,10 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.function.Supplier; -import static org.apache.commons.lang3.StringUtils.isBlank; +import static org.springframework.ai.chat.messages.MessageType.ASSISTANT; public class McpServiceImpl implements McpService @@ -69,12 +64,12 @@ public class McpServiceImpl implements McpService public static final String MESSAGE_ENDPOINT = "/_mcp/message"; public static final String SSE_ENDPOINT = "/_mcp/sse"; - private final CopyOnWriteHashMap toolMap = new CopyOnWriteHashMap<>(); - private final CopyOnWriteHashMap promptMap = new CopyOnWriteHashMap<>(); - private final CopyOnWriteHashMap resourceMap = new CopyOnWriteHashMap<>(); + private final CopyOnWriteHashMap toolMap = new CopyOnWriteHashMap<>(); + private final CopyOnWriteHashMap promptMap = new CopyOnWriteHashMap<>(); + private final CopyOnWriteHashMap resourceMap = new CopyOnWriteHashMap<>(); private final ObjectMapper objectMapper = JsonUtil.DEFAULT_MAPPER; - private final _McpServlet mcpServlet = new _McpServlet(JsonUtil.DEFAULT_MAPPER, MESSAGE_ENDPOINT, SSE_ENDPOINT); + private final _McpServlet mcpServlet = new _McpServlet(JsonUtil.DEFAULT_MAPPER, MESSAGE_ENDPOINT, SSE_ENDPOINT); private final ChatMemoryRepository chatMemoryRepository = new InMemoryChatMemoryRepository(); @@ -246,7 +241,7 @@ public String getParameter(String name) { var ret = super.getParameter(name); if (null == ret && "sessionId".equals(name)) - return String.valueOf(Objects.requireNonNull(((HttpServletRequest)getRequest()).getSession(true).getAttribute("McpServiceImpl#mcpSessionId"))); + return String.valueOf(Objects.requireNonNull(((HttpServletRequest) getRequest()).getSession(true).getAttribute("McpServiceImpl#mcpSessionId"))); return ret; } }; @@ -260,89 +255,6 @@ public Mono closeGracefully() return transportProvider.closeGracefully(); return Mono.empty(); } - - /* - @Override - public Mono closeGracefully() - { - return super.closeGracefully(); - } - - @Override - public void destroy() - { - super.destroy(); - } - - @Override - protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException - { - if (!initialized) - { - response.sendError(HttpServletResponse.SC_SERVICE_UNAVAILABLE); - return; - } - super.doGet(request, response); - } - - @Override - protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException - { - if (!initialized) - { - response.sendError(HttpServletResponse.SC_SERVICE_UNAVAILABLE); - return; - } - - // spring ai requires call to SSE first to get a sessionId???? - if (null == request.getParameter("sessionId")) - { - MockHttpServletRequest mockRequest = new MockHttpServletRequest(request.getServletContext(), "GET", request.getRequestURI()); - MockHttpServletResponse mockResponse = new MockHttpServletResponse(); - doGet(mockRequest, mockResponse); - String body = new String(mockResponse.getContentAsByteArray(), StandardCharsets.UTF_8); - String sessionId = StringUtils.substringBetween(body, "sessionId\":\"", "\""); - request.setAttribute("sessionId", sessionId); - request = new HttpServletRequestWrapper(request) - { - @Override - public String getParameter(String name) - { - if ("sessionId".equals(name)) - return sessionId; - return super.getParameter(name); - } - }; - } - - super.doPost(request, response); - } - - @Override - public Mono notifyClients(String method, Object params) - { - return super.notifyClients(method, params); - } - - @Override - public void setSessionFactory(McpServerSession.Factory sessionFactory) - { - super.setSessionFactory(sessionFactory); - initialized = true; - } - - @Override - protected void service(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException - { - super.service(req, resp); - } - - @Override - public void service(ServletRequest req, ServletResponse res) throws ServletException, IOException - { - super.service(req, res); - } - */ } @@ -387,81 +299,11 @@ String getModel() } - @Override - public Chat getChat(HttpSession session) - { - return SessionHelper.getAttribute(session, Chat.class.getName(), () -> { - Client client = new Client(); - - List fns = new ArrayList<>(); - for (var tc : listTools()) - { - var inputSchema = Schema.fromJson(tc.getToolDefinition().inputSchema()); - var fd = FunctionDeclaration.builder() - .name(tc.getToolDefinition().name()) - .description(tc.getToolDefinition().description()) - .parameters(inputSchema); - fns.add(fd.build()); - } - - GenerateContentConfig config = GenerateContentConfig.builder() - .tools( com.google.genai.types.Tool.builder().functionDeclarations(fns)) - .build(); - Chat chatSession = client.chats.create(getModel(), config); - - return chatSession; - }); - } - - @Override - public String sendMessage(Chat chatSession, String message) - { - // TODO tool context? org.labkey.api.mpc.McpContext.get(); - - GenerateContentResponse response; - List functionCalls; - int sends = 0; - - response = chatSession.sendMessage(message); - sends = sends + 1; - functionCalls = response.functionCalls(); - - while (sends < 3 && null != functionCalls && !functionCalls.isEmpty()) - { - StringBuilder sb = new StringBuilder(); - for (var call : functionCalls) - { - if (call.name().isEmpty()) - break; - var tool = toolMap.get(call.name().get()); - if (null == tool) // ERROR? - continue; - var argsMap = call.args().isEmpty() ? Map.of() : call.args().get(); - var argsString = new JSONObject(argsMap); - String result = tool.call(argsString.toString(), null); - // TODO add context about call and parameters to response? - sb.append(result).append("\n\n"); - } - response = chatSession.sendMessage(sb.toString()); - functionCalls = response.functionCalls(); - } - - // if text is empty and sends > 1 retry the original prompt - var ret = response.text(); - if (isBlank(ret) && sends > 1) - { - response = chatSession.sendMessage(message); - ret = response.text(); - if (isBlank(ret)) - ret = "Too many tool calls. Try again."; - } - return ret; - } // SPRING AI CHAT SERVICE @Override - public ChatClient getChatSpringAi(HttpSession session, String agentName, Supplier systemPromptSupplier) + public ChatClient getChat(HttpSession session, String agentName, Supplier systemPromptSupplier) { return SessionHelper.getAttribute(session, ChatClient.class.getName() + "#" + agentName, () -> { @@ -499,14 +341,24 @@ public ChatClient getChatSpringAi(HttpSession session, String agentName, Supplie @Override - public String sendMessage(ChatClient chatSession, String message) + public MessageResponse sendMessage(ChatClient chatSession, String message) { - String content; - content = chatSession + var callResponse = chatSession .prompt(message) .toolContext(McpContext.get().getToolContext().getContext()) - .call() - .content(); - return content; + .call(); + StringBuilder sb = new StringBuilder(); + for (Generation result : callResponse.chatResponse().getResults()) + { + var output = result.getOutput(); + if (ASSISTANT == output.getMessageType()) + { + sb.append(output.getText()); + sb.append("\n\n"); + } + } + String md = sb.toString().strip(); + HtmlString html = HtmlString.unsafe(MarkdownService.get().toHtml(md)); + return new MessageResponse(md, html); } } diff --git a/devtools/src/org/labkey/devtools/TestController.java b/devtools/src/org/labkey/devtools/TestController.java index be8d2cff08a..601fd27626f 100644 --- a/devtools/src/org/labkey/devtools/TestController.java +++ b/devtools/src/org/labkey/devtools/TestController.java @@ -26,6 +26,7 @@ import org.labkey.api.action.SimpleResponse; import org.labkey.api.action.SimpleViewAction; import org.labkey.api.action.SpringActionController; +import org.labkey.api.mcp.AbstractAgentAction; import org.labkey.api.security.CSRF; import org.labkey.api.security.MethodsAllowed; import org.labkey.api.security.RequiresLogin; @@ -54,6 +55,7 @@ import org.labkey.api.view.UnauthorizedException; import org.labkey.api.view.ViewContext; import org.labkey.api.view.template.ClientDependency; +import org.labkey.api.view.template.PageConfig; import org.springframework.dao.PessimisticLockingFailureException; import org.springframework.validation.BindException; import org.springframework.validation.Errors; @@ -1269,4 +1271,38 @@ public void addNavTrail(NavTree root) { } } + + @RequiresLogin + public static class ChatAction extends SimpleViewAction + { + @Override + public ModelAndView getView(Object o, BindException errors) throws Exception + { + getPageConfig().setTemplate(PageConfig.Template.Dialog); + return new JspView<>("/org/labkey/devtools/view/chat.jsp"); + } + + @Override + public void addNavTrail(NavTree root) + { + root.addChild("chat"); + } + } + + + @RequiresLogin + public static class ChatEndpointAction extends AbstractAgentAction + { + @Override + protected String getAgentName() + { + return "TestController.chat"; + } + + @Override + protected String getServicePrompt() + { + return "You are the generic LabKey agent. Good luck."; + } + } } diff --git a/query/src/org/labkey/query/controllers/QueryController.java b/query/src/org/labkey/query/controllers/QueryController.java index 6207aa43863..8a45fccfee4 100644 --- a/query/src/org/labkey/query/controllers/QueryController.java +++ b/query/src/org/labkey/query/controllers/QueryController.java @@ -8962,7 +8962,7 @@ public Object execute(SqlPromptForm form, BindException errors) throws Exception try { - responseText = McpService.get().sendMessage(chatSession, prompt); + responseText = McpService.get().sendMessage(chatSession, prompt).markdown(); } catch (ServerException x) { @@ -8993,7 +8993,7 @@ public Object execute(SqlPromptForm form, BindException errors) throws Exception catch (QueryException x) { String validationPrompt = "That SQL caused the " + (x instanceof QueryParseWarning ? "warning" : "error") + " below, can you attempt to fix this?\n```" + x.getMessage() + "```"; - responseText = McpService.get().sendMessage(chatSession, validationPrompt); + responseText = McpService.get().sendMessage(chatSession, validationPrompt).markdown(); var newSQL = extractSql(responseText); if (isNotBlank(newSQL)) sql = newSQL; diff --git a/query/src/org/labkey/query/controllers/QueryMcp.java b/query/src/org/labkey/query/controllers/QueryMcp.java index 305f17c65ea..4b480ba220d 100644 --- a/query/src/org/labkey/query/controllers/QueryMcp.java +++ b/query/src/org/labkey/query/controllers/QueryMcp.java @@ -1,7 +1,13 @@ package org.labkey.query.controllers; import io.modelcontextprotocol.server.McpServerFeatures; +import org.apache.commons.lang3.StringUtils; +import org.json.JSONArray; +import org.json.JSONObject; import org.labkey.api.module.McpProvider; +import org.labkey.api.query.DefaultSchema; +import org.labkey.api.view.HttpView; +import org.labkey.api.view.ViewContext; import org.springframework.ai.support.ToolCallbacks; import org.springframework.ai.tool.ToolCallback; import org.springframework.ai.tool.annotation.Tool; @@ -9,6 +15,9 @@ import java.util.Arrays; import java.util.List; +import java.util.Map; + +/* TODO: integrate ToolContext support */ public class QueryMcp implements McpProvider { @@ -46,5 +55,33 @@ String listTablesForSchema(@ToolParam(description = "Fully qualified schema name // can I just return a JSONObject return json.toString(); } + + @Tool(description = "Provide list of database schemas") + String listSchemas() + { + ViewContext context = HttpView.currentView().getViewContext(); + var map = QueryController.listAllSchemas(DefaultSchema.get(context.getUser(), context.getContainer())); + var array = new JSONArray(); + for (var entry : map.entrySet()) + { + array.put(new JSONObject(Map.of( + "name", entry.getKey().getName(), + "quotedName", entry.getKey().toSQLString(), + "description", StringUtils.trimToEmpty(entry.getValue().getDescription()) + ))); + } + return new JSONObject(Map.of("success", "true", "schemas", array)).toString(); + } + + + @Tool(description = "Provide the SQL source for a saved query.") + String getSourceForSaveQuery(@ToolParam(description = "Fully qualified query name as it would appear in SQL e.g. \"schema\".\"saved query\"") String fullQuotedTableName) + { + var json = QueryController.listTablesForSchema(fullQuotedTableName); + if (json.has("sql")) + return "```sql\n" + json.getString("sql") + "\n```\n"; + else + return "I could not find the source for " + fullQuotedTableName; + } } diff --git a/query/src/org/labkey/query/view/sourceQuery.jsp b/query/src/org/labkey/query/view/sourceQuery.jsp index 13e5219fcda..087348bf47a 100644 --- a/query/src/org/labkey/query/view/sourceQuery.jsp +++ b/query/src/org/labkey/query/view/sourceQuery.jsp @@ -83,16 +83,16 @@ border: none; } - /* chat history */ DIV.chatItem { - /* width: 200px; */ + margin: 5px; + padding: 5px; background-color: #4CAF50; border-radius: 15px; + border : solid 1px darkgray; display: flex; - /* justify-content: center; align-items: center; */ - box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1); /* Optional shadow effect */ - } + box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1); + } DIV.userPrompt { margin: 5px; @@ -129,7 +129,7 @@ From f0efa3dea9032ace25aac2be237c38109704ef40 Mon Sep 17 00:00:00 2001 From: labkey-matthewb Date: Wed, 7 Jan 2026 14:35:14 -0800 Subject: [PATCH 19/25] SearchMCP --- api/src/org/labkey/api/mcp/McpService.java | 20 ++++++ .../labkey/api/search/NoopSearchService.java | 6 ++ .../org/labkey/api/search/SearchService.java | 62 ++++++++++++++++--- .../labkey/query/controllers/QueryMcp.java | 31 +--------- .../src/org/labkey/search/SearchModule.java | 7 +++ .../search/model/AbstractSearchService.java | 6 ++ 6 files changed, 94 insertions(+), 38 deletions(-) diff --git a/api/src/org/labkey/api/mcp/McpService.java b/api/src/org/labkey/api/mcp/McpService.java index 802e107f183..1cff5ce61fc 100644 --- a/api/src/org/labkey/api/mcp/McpService.java +++ b/api/src/org/labkey/api/mcp/McpService.java @@ -8,17 +8,23 @@ import org.labkey.api.module.McpProvider; import org.labkey.api.services.ServiceRegistry; import org.labkey.api.util.HtmlString; +import org.springaicommunity.mcp.provider.resource.SyncMcpResourceProvider; import org.springframework.ai.chat.client.ChatClient; +import org.springframework.ai.support.ToolCallbacks; import org.springframework.ai.tool.ToolCallback; import org.springframework.ai.tool.ToolCallbackProvider; import org.springframework.ai.vectorstore.VectorStore; +import java.util.Arrays; import java.util.List; import java.util.function.Supplier; public interface McpService extends ToolCallbackProvider { + // marker interface for classes that we will "injest" using Spring annotations + interface McpImpl {}; + static McpService get() { return ServiceRegistry.get().getService(McpService.class); @@ -31,6 +37,20 @@ static void setInstance(McpService service) boolean isReady(); + + default void register(McpImpl obj) + { + ToolCallback[] tools = ToolCallbacks.from(obj); + if (null != tools && tools.length > 0) + registerTools(Arrays.asList(tools)); + + List ret; + var resources = new SyncMcpResourceProvider(List.of(obj)).getResourceSpecifications(); + if (null != resources && !resources.isEmpty()) + registerResources(resources); + } + + default void register(McpProvider mcp) { registerTools(mcp.getMcpTools()); diff --git a/api/src/org/labkey/api/search/NoopSearchService.java b/api/src/org/labkey/api/search/NoopSearchService.java index fbd402582a5..3b91b203d22 100644 --- a/api/src/org/labkey/api/search/NoopSearchService.java +++ b/api/src/org/labkey/api/search/NoopSearchService.java @@ -414,6 +414,12 @@ public void notFound(URLHelper url) { } + @Override + public List getAllCategories() + { + return List.of(); + } + @Override public List getCategories(String categories) { diff --git a/api/src/org/labkey/api/search/SearchService.java b/api/src/org/labkey/api/search/SearchService.java index 4d64b9ce747..9a1fc3687f6 100644 --- a/api/src/org/labkey/api/search/SearchService.java +++ b/api/src/org/labkey/api/search/SearchService.java @@ -37,6 +37,7 @@ import org.labkey.api.security.permissions.Permission; import org.labkey.api.services.ServiceRegistry; import org.labkey.api.util.DateUtil; +import org.labkey.api.util.GUID; import org.labkey.api.util.Pair; import org.labkey.api.util.Path; import org.labkey.api.util.URLHelper; @@ -53,6 +54,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.Reader; +import java.net.URISyntaxException; import java.sql.SQLException; import java.sql.Timestamp; import java.util.ArrayList; @@ -347,6 +349,26 @@ class SearchHit public JSONObject jsonData; public float score; + public String fullHref(Path contextPath) + { + try + { + ActionURL action = new ActionURL(this.url); + Path path = action.getParsedExtraPath(); + if (path.size() == 1 && GUID.isGUID(action.getParsedExtraPath().getName())) + { + Container c = ContainerManager.getForId(action.getParsedExtraPath().getName()); + if (null != c) + action.setContainer(c); + } + return action.getURIString(false); + } + catch (IllegalArgumentException x) + { + return normalizeHref(contextPath); + } + } + public String normalizeHref(Path contextPath) { Container c = ContainerManager.getForId(container); @@ -362,18 +384,18 @@ public String normalizeHref(Path contextPath, Container c) try { - if (null != c && href.startsWith("/")) - { + if (null != c && href.startsWith("/")) + { URLHelper url = new URLHelper(href); Path path = url.getParsedPath(); - if (path.startsWith(contextPath)) + if (path.startsWith(contextPath)) + { + int pos = path.size() - 2; // look to see if second to last path part is GUID + if (pos>=0 && c.getId().equals(path.get(pos))) { - int pos = path.size() - 2; // look to see if second to last path part is GUID - if (pos>=0 && c.getId().equals(path.get(pos))) - { - path = path.subpath(0,pos) - .append(c.getParsedPath()) - .append(path.subpath(pos+1,path.size())); + path = path.subpath(0,pos) + .append(c.getParsedPath()) + .append(path.subpath(pos+1,path.size())); url.setPath(path); return url.getLocalURIString(false); } @@ -458,6 +480,7 @@ public String normalizeHref(Path contextPath, Container c) // void addSearchCategory(SearchCategory category); + List getAllCategories(); List getCategories(String categories); void addResourceResolver(@NotNull String prefix, @NotNull ResourceResolver resolver); @@ -727,6 +750,27 @@ public Builder(SearchOptions options) this.sortField = options.sortField; } + public Builder limit(int limit) + { + this.limit = limit; + return this; + } + + public Builder scope(SearchScope scope) + { + this.scope = scope; + return this; + } + + /** space separated list */ + public Builder categories(String categories) + { + var list = SearchService.get().getCategories(categories); + if (null != list && !list.isEmpty()) + this.categories = list; + return this; + } + public SearchOptions build() { return new SearchOptions(queryString, user, container, categories, scope, sortField, offset, limit, invertResults, fields); diff --git a/query/src/org/labkey/query/controllers/QueryMcp.java b/query/src/org/labkey/query/controllers/QueryMcp.java index 5285f6b0a97..99bb2df99c0 100644 --- a/query/src/org/labkey/query/controllers/QueryMcp.java +++ b/query/src/org/labkey/query/controllers/QueryMcp.java @@ -1,6 +1,5 @@ package org.labkey.query.controllers; -import io.modelcontextprotocol.server.McpServerFeatures; import io.modelcontextprotocol.spec.McpSchema; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; @@ -14,7 +13,7 @@ import org.labkey.api.data.TableDescription; import org.labkey.api.data.TableInfo; import org.labkey.api.mcp.McpContext; -import org.labkey.api.module.McpProvider; +import org.labkey.api.mcp.McpService; import org.labkey.api.query.DefaultSchema; import org.labkey.api.query.QueryDefinition; import org.labkey.api.query.QueryForeignKey; @@ -26,14 +25,10 @@ import org.labkey.api.security.UserManager; import org.labkey.query.sql.SqlParser; import org.springaicommunity.mcp.annotation.McpResource; -import org.springaicommunity.mcp.provider.resource.SyncMcpResourceProvider; -import org.springframework.ai.support.ToolCallbacks; -import org.springframework.ai.tool.ToolCallback; import org.springframework.ai.tool.annotation.Tool; import org.springframework.ai.tool.annotation.ToolParam; import java.io.IOException; -import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Objects; @@ -43,30 +38,8 @@ /* TODO: integrate ToolContext support */ -public class QueryMcp implements McpProvider +public class QueryMcp implements McpService.McpImpl { - @Override - public List getMcpTools() - { - ToolCallback[] queryTools = ToolCallbacks.from(this); - return Arrays.asList(queryTools); - } - - @Override - public List getMcpPrompts() - { - return List.of(); - } - - @Override - public List getMcpResources() - { - List ret; - ret = new SyncMcpResourceProvider(List.of(this)).getResourceSpecifications(); - return ret; - } - - @McpResource( uri = "resource://org/labkey/query/controllers/LabKeySql.md", mimeType = "application/markdown", diff --git a/search/src/org/labkey/search/SearchModule.java b/search/src/org/labkey/search/SearchModule.java index 1a93547b925..6305b9dae18 100644 --- a/search/src/org/labkey/search/SearchModule.java +++ b/search/src/org/labkey/search/SearchModule.java @@ -29,6 +29,7 @@ import org.labkey.api.data.UpgradeCode; import org.labkey.api.mbean.LabKeyManagement; import org.labkey.api.mbean.SearchMXBean; +import org.labkey.api.mcp.McpService; import org.labkey.api.migration.DatabaseMigrationConfiguration; import org.labkey.api.migration.DatabaseMigrationService; import org.labkey.api.migration.DefaultMigrationSchemaHandler; @@ -132,6 +133,12 @@ public WebdavResource resolve(@NotNull String path) return WebdavService.get().lookup(path); } }); + + var mcp = McpService.get(); + if (null != mcp) + { + mcp.register(new SearchMcp()); + } } @Override diff --git a/search/src/org/labkey/search/model/AbstractSearchService.java b/search/src/org/labkey/search/model/AbstractSearchService.java index eadde97fbd9..92dc036e571 100644 --- a/search/src/org/labkey/search/model/AbstractSearchService.java +++ b/search/src/org/labkey/search/model/AbstractSearchService.java @@ -1376,6 +1376,12 @@ public void addSearchCategory(SearchCategory category) } } + @Override + public List getAllCategories() + { + return _readonlyCategories; + } + @Override public List getCategories(String categories) { From 7d166aa04659b134b29a1503f8285465b9205225 Mon Sep 17 00:00:00 2001 From: labkey-matthewb Date: Wed, 7 Jan 2026 16:49:20 -0800 Subject: [PATCH 20/25] SearchMCP, CoreMcp --- core/src/org/labkey/core/CoreMcp.java | 66 ++++++++++ core/src/org/labkey/core/CoreModule.java | 3 + search/src/org/labkey/search/SearchMcp.java | 132 ++++++++++++++++++++ 3 files changed, 201 insertions(+) create mode 100644 core/src/org/labkey/core/CoreMcp.java create mode 100644 search/src/org/labkey/search/SearchMcp.java diff --git a/core/src/org/labkey/core/CoreMcp.java b/core/src/org/labkey/core/CoreMcp.java new file mode 100644 index 00000000000..06482fe5627 --- /dev/null +++ b/core/src/org/labkey/core/CoreMcp.java @@ -0,0 +1,66 @@ +package org.labkey.core; + +import org.json.JSONObject; +import org.labkey.api.data.Container; +import org.labkey.api.mcp.McpContext; +import org.labkey.api.mcp.McpService; +import org.labkey.api.security.User; +import org.labkey.api.settings.AppProps; +import org.labkey.api.settings.LookAndFeelProperties; +import org.labkey.api.study.Study; +import org.labkey.api.study.StudyService; +import org.labkey.api.util.HtmlString; +import org.springframework.ai.tool.annotation.Tool; + +import java.util.Map; +import java.util.Objects; + +import static org.apache.commons.lang3.StringUtils.isNotBlank; + +public class CoreMcp implements McpService.McpImpl +{ + // TODO ChatSessions are currently per session. The McpService should detect change of folder. + @Tool(description = "Call this tool before answering any prompts! This tool provides useful context information about the current user (name, userid), webserver (name, url, description), and current folder (name, path, url, description).") + String whereAmIWhoAmITalkingTo() + { + McpContext context = McpContext.get(); + User user = context.getUser(); + Container folder = context.getContainer(); + AppProps appProps = AppProps.getInstance(); + Study study = null != StudyService.get() ? Objects.requireNonNull(StudyService.get()).getStudy(folder) : null; + LookAndFeelProperties laf = LookAndFeelProperties.getInstance(folder); + + JSONObject userObj = new JSONObject(); + userObj.put("userId", user.getUserId()); + userObj.put("displayName", user.getDisplayName(user)); + if (isNotBlank(user.getFirstName())) + userObj.put("firstName", user.getFirstName()); + + JSONObject folderObj = new JSONObject(); + folderObj.put("name", folder.getName()); + folderObj.put("path", folder.getPath()); + folderObj.put("startUrl", folder.getStartURL(user).getURIString()); + if (isNotBlank(folder.getDescription())) + folderObj.put("description", folder.getDescription()); + if (null != study) + { + var studyDescription = study.getDescriptionHtml(); + if (!HtmlString.isBlank(studyDescription)) + { + folderObj.put("studyDescription", new JSONObject(Map.of("contentType", "text/html", "content", studyDescription.toString()))); + } + } + + JSONObject siteObj = new JSONObject(); + siteObj.put("name", appProps.getServerName()); + siteObj.put("baseServerUrl", appProps.getBaseServerUrl()); + siteObj.put("description", laf.getDescription()); + siteObj.put("homePageUrl", appProps.getHomePageUrl()); + + return new JSONObject(Map.of( + "user", userObj, + "currentFolder", folderObj, + "site", siteObj + )).toString(); + } +} diff --git a/core/src/org/labkey/core/CoreModule.java b/core/src/org/labkey/core/CoreModule.java index a2ede2ac74d..ce6b69dab9c 100644 --- a/core/src/org/labkey/core/CoreModule.java +++ b/core/src/org/labkey/core/CoreModule.java @@ -561,9 +561,12 @@ public QuerySchema createSchema(DefaultSchema schema, Module module) ScriptEngineManagerImpl.registerEncryptionMigrationHandler(); + McpService.get().register(new CoreMcp()); + deleteTempFiles(); } + private void deleteTempFiles() { try diff --git a/search/src/org/labkey/search/SearchMcp.java b/search/src/org/labkey/search/SearchMcp.java new file mode 100644 index 00000000000..d896830d8e0 --- /dev/null +++ b/search/src/org/labkey/search/SearchMcp.java @@ -0,0 +1,132 @@ +package org.labkey.search; + +import org.apache.commons.lang3.StringUtils; +import org.json.JSONArray; +import org.json.JSONObject; +import org.labkey.api.mcp.McpContext; +import org.labkey.api.mcp.McpService; +import org.labkey.api.search.SearchScope; +import org.labkey.api.search.SearchService; +import org.labkey.api.settings.AppProps; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.util.Path; +import org.springframework.ai.tool.annotation.Tool; +import org.springframework.ai.tool.annotation.ToolParam; + +import java.io.IOException; +import java.util.Map; + +import static org.apache.commons.lang3.StringUtils.isBlank; +import static org.apache.commons.lang3.StringUtils.trimToEmpty; + +public class SearchMcp implements McpService.McpImpl +{ + final static String mdSearchHelp = """ + The search functionality is implmeneted by Lucene. The query syntax is + + Core Syntax Elements + + * **Terms and Phrases**: + * A **term** is a single word (e.g., `error`). + * A **phrase** is a group of words surrounded by double quotes (e.g., `"network error"`), which searches for all words in the specified order. + * **Fields**: You can search specific fields using the format `fieldName:searchValue` (e.g., `title:malaria` or `user:"John Doe"`). + * **Grouping**: Use parentheses `()` to group clauses and control Boolean logic (e.g., `title:(apple OR pie) AND description:apple`). + + Boolean Operators + + Boolean operators must be in **ALL CAPS**. + + * `AND` (`&&`, `+`): Requires both terms to be present (e.g., `wifi AND luxury` or `+wifi +luxury`). + * `OR` (`||`): Requires at least one term to be present (default operator if none is specified) (e.g., `wifi OR luxury`). + * `NOT` (`!`, `-`): Excludes documents that contain the term after the operator (e.g., `wifi NOT luxury` or `wifi -luxury`). + + Term Modifiers + + * **Wildcard Searches**: + * `?` for a single character (e.g., `te?t` matches "test" or "text"). + * `*` for multiple characters (zero or more) (e.g., `test*` matches "test", "tests", "tester"). + * _Note_: You cannot use `*` or `?` as the first character of a search term. + * **Fuzzy Searches**: Use the tilde `~` symbol at the end of a single word to find terms with a similar spelling (e.g., `roam~` finds "foam" and "roams"). An optional number between 0 and 2 can specify the required similarity (default is 0.5). + * **Proximity Searches**: Use the tilde `~` at the end of a phrase to find words within a specific distance (e.g., `"jakarta apache"~10` finds "jakarta" and "apache" within 10 words of each other). + * **Range Searches**: Match documents whose field values are between a lower and upper bound. + * Inclusive (square brackets): `mod_date:[20020101 TO 20030101]`. + * Exclusive (curly brackets): `title:{Aida TO Carmen}`. + * One-sided range: `score:[2.5 TO *]`. + * **Boosting Terms**: Use the caret `^` symbol with a numerical boost factor to increase the relevance of a term (e.g., `jakarta^4 apache` makes "jakarta" more relevant). + + Escaping Special Characters + + To use a special character as part of your search text, escape it with a single backslash `\\`. Special characters include: + `+ - && || ! ( ) { } [ ] ^ " ~ * ? : \\ /` + """; + + @Tool(description = "Search this LabKey server. This may be useful for site navigation purposes. When rendering results that use this tool present full URLs whenever relevant.") + String siteSearch( + @ToolParam(description = mdSearchHelp) String query, + @ToolParam(required=false, description="comma separated list of categories, use category=navigation to find folders/projects/studies. use the listSearchCategories tool to find other options.") String categories + ) + { + SearchService ss = SearchService.get(); + if (isBlank(query)) + { + return new JSONObject(Map.of( + "success", Boolean.TRUE, + "hits", new JSONArray())).toString(); + } + + var list = PageFlowUtil.splitStringToValuesForImport(trimToEmpty(categories)); + + McpContext context = McpContext.get(); + Path contextPath = AppProps.getInstance().getParsedContextPath(); + var options = new SearchService.SearchOptions.Builder(query,context.getUser(),context.getContainer()) + .limit(20) + .scope(SearchScope.All); + if (!list.isEmpty()) + options.categories(StringUtils.join(list,' ')); + + try + { + JSONArray hits = new JSONArray(); + var searchResult = ss.search(options.build()); + for (var hit : searchResult.hits) + { + JSONObject o = new JSONObject(); + o.put("title", hit.title); + o.put("container", hit.container); + o.put("url", hit.fullHref(contextPath)); + o.put("summary", trimToEmpty(hit.summary)); + o.put("score", hit.score); + o.put("identifiers", hit.identifiers); + o.put("category", trimToEmpty(hit.category)); + hits.put(o); + } + return new JSONObject(Map.of( + "success", Boolean.TRUE, + "baseServerUrl", AppProps.getInstance().getBaseServerUrl(), + "hits", hits, + "totalHits", searchResult.totalHits + )).toString(); + } + catch (IOException io) + { + return new JSONObject(Map.of( + "success", Boolean.FALSE, + "error", io.getMessage() + )).toString(); + } + } + + @Tool(description = "Return list of valid categories for the siteSearch tool") + String listSearchCategories() + { + JSONArray list = new JSONArray(); + for (var cat : SearchService.get().getAllCategories()) + { + list.put(cat.getName()); + } + return new JSONObject(Map.of( + "success", Boolean.TRUE, + "categories", list + )).toString(); + } +} From 17f040795664e8a1ef0e4fec4c5ca9e30ba90630 Mon Sep 17 00:00:00 2001 From: labkey-matthewb Date: Wed, 14 Jan 2026 11:12:13 -0800 Subject: [PATCH 21/25] <% if (isChatReady) { %> --- .../org/labkey/core/mpc/McpServiceImpl.java | 3 ++ .../src/org/labkey/query/view/sourceQuery.jsp | 29 ++++++++++--------- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/core/src/org/labkey/core/mpc/McpServiceImpl.java b/core/src/org/labkey/core/mpc/McpServiceImpl.java index 6a9ba7bc857..ac4dc7fed78 100644 --- a/core/src/org/labkey/core/mpc/McpServiceImpl.java +++ b/core/src/org/labkey/core/mpc/McpServiceImpl.java @@ -120,6 +120,9 @@ String bye() public void startMpcServer() { + /* For now the presense of GEMINI_API_KEY will enable/disable the McpServer */ + if (isBlank(System.getenv("GEMINI_API_KEY"))) + return; vectorStore = createVectorStore(); mcpServlet.startMcpServer(); serverReady = true; diff --git a/query/src/org/labkey/query/view/sourceQuery.jsp b/query/src/org/labkey/query/view/sourceQuery.jsp index 2bc380a885b..c12fa995bcd 100644 --- a/query/src/org/labkey/query/view/sourceQuery.jsp +++ b/query/src/org/labkey/query/view/sourceQuery.jsp @@ -53,7 +53,6 @@ boolean canEdit = queryDef.canEdit(getUser()); boolean canEditMetadata = queryDef.canEditMetadata(getUser()); boolean canDelete = queryDef.canDelete(getUser()); - boolean isChatReady = McpService.get().isReady(); %> -<%-- should use Ext4 Panel for layout, but this is just a prototype anyway --%> <% if (isChatReady) { %> +<%-- should use Ext4 Panel for layout, but this is just a prototype anyway --%>
@@ -129,14 +129,13 @@
+<% } else { %> + +
<% } %> From 837e1ab2f65e1e9c3f7ab58d5b8bbc059c5d467e Mon Sep 17 00:00:00 2001 From: labkey-matthewb Date: Wed, 14 Jan 2026 11:22:24 -0800 Subject: [PATCH 22/25] comment --- .../query/controllers/QueryController.java | 81 +------------------ 1 file changed, 1 insertion(+), 80 deletions(-) diff --git a/query/src/org/labkey/query/controllers/QueryController.java b/query/src/org/labkey/query/controllers/QueryController.java index e3effe7ba50..96ddc095e92 100644 --- a/query/src/org/labkey/query/controllers/QueryController.java +++ b/query/src/org/labkey/query/controllers/QueryController.java @@ -8805,69 +8805,6 @@ private JSONArray getTestRows(String val) } - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - public static class QueryWriterAction extends SimpleViewAction - { - @Override - public ModelAndView getView(Object o, BindException errors) throws Exception - { - return new JspView<>("/org/labkey/query/view/queryWriter.jsp", null, errors); - } - - @Override - public void addNavTrail(NavTree root) - { - - } - } - - public static class SqlPromptForm extends PromptForm { public String schemaName; @@ -8923,7 +8860,6 @@ protected String getServicePrompt() return serviceMessage.toString(); } - String getSQLHelp() { try @@ -8944,6 +8880,7 @@ public Object execute(SqlPromptForm form, BindException errors) throws Exception try (var mcpPush = McpContext.withContext(getViewContext())) { + // TODO when/how to do we reset or isolate different chat sessions, e.g. if two SQL windows are open concurrently? ChatClient chatSession = getChat(); String prompt = form.getPrompt(); List responses; @@ -8994,7 +8931,6 @@ public Object execute(SqlPromptForm form, BindException errors) throws Exception } } -// System.err.println(chatSession.getHistory(true)); var ret = new JSONObject(Map.of( "success", Boolean.TRUE)); if (null != sqlResponse.sql()) @@ -9058,19 +8994,4 @@ static String extractSql(String text) } return null; } - - - @RequiresPermission(ReadPermission.class) - @RequiresLogin - public static class ResetQueryAgentAction extends ReadOnlyApiAction - { - @Override - public Object execute(Object o, BindException errors) throws Exception - { - var session = getViewContext().getRequest().getSession(false); - if (null != session) - session.removeAttribute(Chat.class.getName()); - return new JSONObject(Map.of("success", Boolean.TRUE)); - } - } } From 8ac79eb2e3c1d7d387c4e2f1d9fd1d825f6c6a82 Mon Sep 17 00:00:00 2001 From: labkey-matthewb Date: Thu, 15 Jan 2026 17:09:22 -0800 Subject: [PATCH 23/25] remove unused dependency (moved to api) --- query/build.gradle | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/query/build.gradle b/query/build.gradle index 02aa43e8afe..33289e9b980 100644 --- a/query/build.gradle +++ b/query/build.gradle @@ -27,19 +27,6 @@ dependencies { ) ) - BuildUtils.addExternalDependency( - project, - new ExternalDependency( - "com.google.genai:google-genai:1.15.0", - "GENAI", - "GENAI", - "https://google.com/", - "???", - "???", - "GenAI" - ) - ) - BuildUtils.addExternalDependency( project, new ExternalDependency( From 7669e93e913062ca3fd0b085866c7caa80740ff0 Mon Sep 17 00:00:00 2001 From: labkey-matthewb Date: Thu, 15 Jan 2026 17:37:16 -0800 Subject: [PATCH 24/25] CR cleanup --- api/src/org/labkey/api/mcp/McpService.java | 4 ++-- api/src/org/labkey/api/module/McpProvider.java | 15 ++++++++++++--- core/src/org/labkey/core/mpc/McpServiceImpl.java | 4 +++- .../src/org/labkey/devtools/TestController.java | 7 ++++--- query/src/org/labkey/query/QueryModule.java | 6 +----- 5 files changed, 22 insertions(+), 14 deletions(-) diff --git a/api/src/org/labkey/api/mcp/McpService.java b/api/src/org/labkey/api/mcp/McpService.java index 1cff5ce61fc..4e70e95df1d 100644 --- a/api/src/org/labkey/api/mcp/McpService.java +++ b/api/src/org/labkey/api/mcp/McpService.java @@ -22,10 +22,10 @@ public interface McpService extends ToolCallbackProvider { - // marker interface for classes that we will "injest" using Spring annotations + // marker interface for classes that we will "ingest" using Spring annotations interface McpImpl {}; - static McpService get() + static @NotNull McpService get() { return ServiceRegistry.get().getService(McpService.class); } diff --git a/api/src/org/labkey/api/module/McpProvider.java b/api/src/org/labkey/api/module/McpProvider.java index 1e42582ab53..6ca9061e73f 100644 --- a/api/src/org/labkey/api/module/McpProvider.java +++ b/api/src/org/labkey/api/module/McpProvider.java @@ -7,9 +7,18 @@ public interface McpProvider { - List getMcpTools(); + default List getMcpTools() + { + return List.of(); + } - List getMcpPrompts(); + default List getMcpPrompts() + { + return List.of(); + } - List getMcpResources(); + default List getMcpResources() + { + return List.of(); + } } diff --git a/core/src/org/labkey/core/mpc/McpServiceImpl.java b/core/src/org/labkey/core/mpc/McpServiceImpl.java index ac4dc7fed78..b752e808032 100644 --- a/core/src/org/labkey/core/mpc/McpServiceImpl.java +++ b/core/src/org/labkey/core/mpc/McpServiceImpl.java @@ -31,6 +31,7 @@ import org.labkey.api.util.JsonUtil; import org.labkey.api.util.SessionHelper; import org.labkey.api.util.ShutdownListener; +import org.labkey.api.util.logging.LogHelper; import org.springframework.ai.chat.client.ChatClient; import org.springframework.ai.chat.client.advisor.MessageChatMemoryAdvisor; import org.springframework.ai.chat.client.advisor.vectorstore.QuestionAnswerAdvisor; @@ -444,7 +445,8 @@ private VectorStore createVectorStore() } catch (Exception x) { - System.err.println(x.getMessage()); + LogHelper.getLogger(McpServiceImpl.class,"mcp service") + .error("error restoring saved vectordb: " + savedFile.toNioPathForRead(), x); } } } diff --git a/devtools/src/org/labkey/devtools/TestController.java b/devtools/src/org/labkey/devtools/TestController.java index 69c9dbe2d37..09f7fe0ef9b 100644 --- a/devtools/src/org/labkey/devtools/TestController.java +++ b/devtools/src/org/labkey/devtools/TestController.java @@ -71,6 +71,7 @@ import org.springframework.dao.PessimisticLockingFailureException; import org.springframework.validation.BindException; import org.springframework.validation.Errors; +import org.springframework.validation.ObjectError; import org.springframework.web.servlet.ModelAndView; import org.springframework.web.servlet.mvc.Controller; @@ -1394,13 +1395,13 @@ public boolean handlePost(Object o, BindException errors) throws Exception try { ((SimpleVectorStore)vs).save(db.toNioPathForRead().toFile()); + return true; } catch (Exception x) { - System.err.println(x.getMessage()); + errors.addError(new ObjectError("form", "error saving vectordb: " + x.getMessage())); + return false; } - - return true; } } } diff --git a/query/src/org/labkey/query/QueryModule.java b/query/src/org/labkey/query/QueryModule.java index 6f07cd505a4..b9a63544ada 100644 --- a/query/src/org/labkey/query/QueryModule.java +++ b/query/src/org/labkey/query/QueryModule.java @@ -240,11 +240,7 @@ public QuerySchema createSchema(DefaultSchema schema, Module module) OptionalFeatureService.get().addExperimentalFeatureFlag(QueryServiceImpl.EXPERIMENTAL_PRODUCT_PROJECT_DATA_LISTING_SCOPED, "Product folders display folder-specific data", "Only list folder-specific data within product folders.", false); - var mcp = McpService.get(); - if (null != mcp) - { - mcp.register(new QueryMcp()); - } + McpService.get().register(new QueryMcp()); } From a7ce75f78e364713c79fbb9e64e3377928860be9 Mon Sep 17 00:00:00 2001 From: labkey-matthewb Date: Thu, 15 Jan 2026 17:50:31 -0800 Subject: [PATCH 25/25] comments --- api/src/org/labkey/api/mcp/AbstractAgentAction.java | 6 ++++++ api/src/org/labkey/api/mcp/McpContext.java | 5 +++++ api/src/org/labkey/api/mcp/McpService.java | 10 +++++++++- 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/api/src/org/labkey/api/mcp/AbstractAgentAction.java b/api/src/org/labkey/api/mcp/AbstractAgentAction.java index b0332cc2069..91dddcc7eaf 100644 --- a/api/src/org/labkey/api/mcp/AbstractAgentAction.java +++ b/api/src/org/labkey/api/mcp/AbstractAgentAction.java @@ -13,6 +13,12 @@ import static org.apache.commons.lang3.StringUtils.isNotBlank; +/** + * "agent" it is too strong a word, but if you want to create a tools specific chat endpoint then + * start here. + * First implement getServicePrompt() to tell your "agent its mission. You can also listen in on the + * conversation to help you user get the right results. + */ public abstract class AbstractAgentAction extends ReadOnlyApiAction { protected abstract String getAgentName(); diff --git a/api/src/org/labkey/api/mcp/McpContext.java b/api/src/org/labkey/api/mcp/McpContext.java index af924e097e2..d1b0975f46f 100644 --- a/api/src/org/labkey/api/mcp/McpContext.java +++ b/api/src/org/labkey/api/mcp/McpContext.java @@ -9,6 +9,11 @@ import org.springframework.ai.chat.model.ToolContext; import java.util.Map; +/** + * TODO MCP tool calling supports passing along a ToolContext. And most all + * interesting tools probably need a User and Container. This is not all hooked-up + * yet. This is an area for further investiation. + */ public class McpContext implements ContainerUser { final User user; diff --git a/api/src/org/labkey/api/mcp/McpService.java b/api/src/org/labkey/api/mcp/McpService.java index 4e70e95df1d..ba22c53d545 100644 --- a/api/src/org/labkey/api/mcp/McpService.java +++ b/api/src/org/labkey/api/mcp/McpService.java @@ -19,7 +19,15 @@ import java.util.List; import java.util.function.Supplier; - +/** + * This service lets you expose functionality over the MCP protocol (only simple http for now). This allows + * external chat sessions to pull information from LabKey Server. These methods are also made available + * to chat session shosted by LabKey (see AbstractAgentAction). + *

+ * These calls are not security checked. Any tools registered here must check user permissions. Maybe that + * will come as we get further along. Note that the LLM may make callbacks concerning containers other than the + * current container. This is an area for investigation. + */ public interface McpService extends ToolCallbackProvider { // marker interface for classes that we will "ingest" using Spring annotations