diff --git a/EmbeddedSQLTester/EmbeddedSQLTester.csproj b/EmbeddedSQLTester/EmbeddedSQLTester.csproj index 9fce12a..31f08c4 100644 --- a/EmbeddedSQLTester/EmbeddedSQLTester.csproj +++ b/EmbeddedSQLTester/EmbeddedSQLTester.csproj @@ -10,7 +10,7 @@ - + diff --git a/EmbeddedSQLTester/SQLitePlatformConversion/SQLServerToOrmliteSQLiteDialectConverter.cs b/EmbeddedSQLTester/SQLitePlatformConversion/SQLServerToOrmliteSQLiteDialectConverter.cs index fb4044a..93f2554 100644 --- a/EmbeddedSQLTester/SQLitePlatformConversion/SQLServerToOrmliteSQLiteDialectConverter.cs +++ b/EmbeddedSQLTester/SQLitePlatformConversion/SQLServerToOrmliteSQLiteDialectConverter.cs @@ -1,8 +1,10 @@ using System.Collections.Generic; +using System.IO; using System.Text; using EmbeddedSQLTester.SQLitePlatformConversion.StatementConverters; -using TSQL; -using TSQL.Statements; +using EmbeddedSQLTester.SQLitePlatformConversion.StatementConverters.ClauseProcessors; +using EmbeddedSQLTester.SQLitePlatformConversion.StatementConverters.ExpressionProcessors; +using Microsoft.SqlServer.TransactSql.ScriptDom; namespace EmbeddedSQLTester.SQLitePlatformConversion { @@ -13,33 +15,47 @@ public class SQLServerToOrmliteSQLiteDialectConverter public string ConvertToOrmliteSQLiteSQL(string sqlInput) { - var statements = TSQLStatementReader.ParseStatements(sqlInput); + var parser = new TSql150Parser(false); + using (var reader = new StringReader(sqlInput)) + { + var fragment = parser.Parse(reader, out var errors); + var script = (TSqlScript)fragment; - var statement = statements[0]; + if (script.Batches.Count > 0 && script.Batches[0].Statements.Count > 0) + { + // AST parsing succeeded — use structured approach + var statement = script.Batches[0].Statements[0]; - if (statement.GetType() == typeof(TSQLSelectStatement)) - { - var selectStatement = (TSQLSelectStatement)statement; - var converter = new SelectStatementConverter(); - _clauseConverters = converter.GetClauseProcessorList(selectStatement); - } - else if (statement.GetType() == typeof(TSQLUpdateStatement)) - { - var updateStatement = (TSQLUpdateStatement)statement; - var converter = new UpdateStatementConverter(); - _clauseConverters = converter.GetClauseProcessorList(updateStatement); - } - else if (statement.GetType() == typeof(TSQLInsertStatement)) - { - var insertStatement = (TSQLInsertStatement)statement; - var converter = new InsertStatementConverter(); - _clauseConverters = converter.GetClauseProcessorList(insertStatement); - } - else if (statement.GetType() == typeof(TSQLDeleteStatement)) - { - var deleteStatement = (TSQLDeleteStatement)statement; - var converter = new DeleteStatementConverter(); - _clauseConverters = converter.GetClauseProcessorList(deleteStatement); + if (statement is SelectStatement selectStatement) + { + var converter = new SelectStatementConverter(); + _clauseConverters = converter.GetClauseProcessorList(selectStatement); + } + else if (statement is UpdateStatement updateStatement) + { + var converter = new UpdateStatementConverter(); + _clauseConverters = converter.GetClauseProcessorList(updateStatement); + } + else if (statement is InsertStatement insertStatement) + { + var converter = new InsertStatementConverter(); + _clauseConverters = converter.GetClauseProcessorList(insertStatement); + } + else if (statement is DeleteStatement deleteStatement) + { + var converter = new DeleteStatementConverter(); + _clauseConverters = converter.GetClauseProcessorList(deleteStatement); + } + } + else + { + // AST parsing failed — fall back to token-based clause splitting + var stream = fragment.ScriptTokenStream; + var stmtType = TokenBasedStatementParser.DetectStatementType(stream); + var clauses = TokenBasedStatementParser.SplitIntoClauses(stream, stmtType); + + _clauseConverters = BuildConvertersFromClauses(stmtType, clauses); + } } var sb = new StringBuilder(); @@ -48,6 +64,47 @@ public string ConvertToOrmliteSQLiteSQL(string sqlInput) sb.Append(clauseConverter.Convert()); return sb.ToString(); - } + } + + private List BuildConvertersFromClauses(string stmtType, + Dictionary> clauses) + { + switch (stmtType) + { + case "SELECT": + return new List + { + new GeneralClauseConverter(clauses.ContainsKey("SELECT") ? clauses["SELECT"] : null), + new FromClauseConverter(clauses.ContainsKey("FROM") ? clauses["FROM"] : null), + new WhereClauseConverter(clauses.ContainsKey("WHERE") ? clauses["WHERE"] : null), + new GeneralClauseConverter(clauses.ContainsKey("GROUPBY") ? clauses["GROUPBY"] : null), + new OrderByClauseConverter(clauses.ContainsKey("ORDERBY") ? clauses["ORDERBY"] : null), + new LimitStatementConverter(clauses.ContainsKey("SELECT") ? clauses["SELECT"] : null), + }; + case "UPDATE": + return new List + { + new UpdateClauseConverter(clauses.ContainsKey("UPDATE") ? clauses["UPDATE"] : null), + new GeneralClauseConverter(clauses.ContainsKey("SET") ? clauses["SET"] : null), + new FromClauseConverter(clauses.ContainsKey("FROM") ? clauses["FROM"] : null), + new WhereClauseConverter(clauses.ContainsKey("WHERE") ? clauses["WHERE"] : null), + }; + case "INSERT": + return new List + { + new InsertClauseConverter(clauses.ContainsKey("INSERT") ? clauses["INSERT"] : null), + new ValuesExpressionConverter(clauses.ContainsKey("VALUES") ? clauses["VALUES"] : null), + }; + case "DELETE": + return new List + { + new GeneralClauseConverter(clauses.ContainsKey("DELETE") ? clauses["DELETE"] : null), + new FromClauseConverter(clauses.ContainsKey("FROM") ? clauses["FROM"] : null), + new WhereClauseConverter(clauses.ContainsKey("WHERE") ? clauses["WHERE"] : null), + }; + default: + return new List(); + } + } } } \ No newline at end of file diff --git a/EmbeddedSQLTester/SQLitePlatformConversion/StatementConverters/ClauseProcessors/ClauseConverterBase.cs b/EmbeddedSQLTester/SQLitePlatformConversion/StatementConverters/ClauseProcessors/ClauseConverterBase.cs index 990d7f7..59e992a 100644 --- a/EmbeddedSQLTester/SQLitePlatformConversion/StatementConverters/ClauseProcessors/ClauseConverterBase.cs +++ b/EmbeddedSQLTester/SQLitePlatformConversion/StatementConverters/ClauseProcessors/ClauseConverterBase.cs @@ -1,28 +1,26 @@ using System.Collections.Generic; using System.Text; -using TSQL.Clauses; -using TSQL.Tokens; namespace EmbeddedSQLTester.SQLitePlatformConversion.StatementConverters.ClauseProcessors { internal abstract class ClauseConverterBase : ConverterBase { - private readonly TSQLClause _clause; + private readonly List _tokens; private StringBuilder _stringBuilder; - protected List Tokens; + protected List Tokens; - protected ClauseConverterBase(TSQLClause clauseParameter) + protected ClauseConverterBase(List tokens) { - _clause = clauseParameter; + _tokens = tokens; } public override string Convert() { _stringBuilder = new StringBuilder(); - if (_clause != null) + if (_tokens != null) { - Tokens = _clause.Tokens; + Tokens = _tokens; ConvertHelper(); } diff --git a/EmbeddedSQLTester/SQLitePlatformConversion/StatementConverters/ClauseProcessors/FromClauseConverter.cs b/EmbeddedSQLTester/SQLitePlatformConversion/StatementConverters/ClauseProcessors/FromClauseConverter.cs index d55883d..ba7d139 100644 --- a/EmbeddedSQLTester/SQLitePlatformConversion/StatementConverters/ClauseProcessors/FromClauseConverter.cs +++ b/EmbeddedSQLTester/SQLitePlatformConversion/StatementConverters/ClauseProcessors/FromClauseConverter.cs @@ -1,4 +1,4 @@ -using TSQL.Clauses; +using System.Collections.Generic; namespace EmbeddedSQLTester.SQLitePlatformConversion.StatementConverters.ClauseProcessors { @@ -6,7 +6,7 @@ internal class FromClauseConverter : ClauseConverterBase { private bool _insideOnClause; - public FromClauseConverter(TSQLClause clause) : base(clause) + public FromClauseConverter(List tokens) : base(tokens) { _insideOnClause = false; } diff --git a/EmbeddedSQLTester/SQLitePlatformConversion/StatementConverters/ClauseProcessors/GeneralClauseConverter.cs b/EmbeddedSQLTester/SQLitePlatformConversion/StatementConverters/ClauseProcessors/GeneralClauseConverter.cs index 2acdedf..f83604d 100644 --- a/EmbeddedSQLTester/SQLitePlatformConversion/StatementConverters/ClauseProcessors/GeneralClauseConverter.cs +++ b/EmbeddedSQLTester/SQLitePlatformConversion/StatementConverters/ClauseProcessors/GeneralClauseConverter.cs @@ -1,13 +1,12 @@ using System; -using System.Net.Mime; +using System.Collections.Generic; using System.Text; -using TSQL.Clauses; namespace EmbeddedSQLTester.SQLitePlatformConversion.StatementConverters.ClauseProcessors { internal class GeneralClauseConverter : ClauseConverterBase { - public GeneralClauseConverter(TSQLClause clause) : base(clause) + public GeneralClauseConverter(List tokens) : base(tokens) { } diff --git a/EmbeddedSQLTester/SQLitePlatformConversion/StatementConverters/ClauseProcessors/InsertClauseConverter.cs b/EmbeddedSQLTester/SQLitePlatformConversion/StatementConverters/ClauseProcessors/InsertClauseConverter.cs index 201aaac..22dbc94 100644 --- a/EmbeddedSQLTester/SQLitePlatformConversion/StatementConverters/ClauseProcessors/InsertClauseConverter.cs +++ b/EmbeddedSQLTester/SQLitePlatformConversion/StatementConverters/ClauseProcessors/InsertClauseConverter.cs @@ -1,11 +1,10 @@ -using System.Text; -using TSQL.Clauses; +using System.Collections.Generic; namespace EmbeddedSQLTester.SQLitePlatformConversion.StatementConverters.ClauseProcessors { internal class InsertClauseConverter : ClauseConverterBase { - public InsertClauseConverter(TSQLClause clause) : base(clause) + public InsertClauseConverter(List tokens) : base(tokens) { } diff --git a/EmbeddedSQLTester/SQLitePlatformConversion/StatementConverters/ClauseProcessors/OrderByClauseConverter.cs b/EmbeddedSQLTester/SQLitePlatformConversion/StatementConverters/ClauseProcessors/OrderByClauseConverter.cs index 9713f1f..c1581d0 100644 --- a/EmbeddedSQLTester/SQLitePlatformConversion/StatementConverters/ClauseProcessors/OrderByClauseConverter.cs +++ b/EmbeddedSQLTester/SQLitePlatformConversion/StatementConverters/ClauseProcessors/OrderByClauseConverter.cs @@ -1,11 +1,11 @@ +using System.Collections.Generic; using System.Text; -using TSQL.Clauses; namespace EmbeddedSQLTester.SQLitePlatformConversion.StatementConverters.ClauseProcessors { internal class OrderByClauseConverter : ClauseConverterBase { - public OrderByClauseConverter(TSQLClause clauseParameter) : base(clauseParameter) + public OrderByClauseConverter(List tokens) : base(tokens) { } diff --git a/EmbeddedSQLTester/SQLitePlatformConversion/StatementConverters/ClauseProcessors/UpdateClauseConverter.cs b/EmbeddedSQLTester/SQLitePlatformConversion/StatementConverters/ClauseProcessors/UpdateClauseConverter.cs index 1e88819..e171b06 100644 --- a/EmbeddedSQLTester/SQLitePlatformConversion/StatementConverters/ClauseProcessors/UpdateClauseConverter.cs +++ b/EmbeddedSQLTester/SQLitePlatformConversion/StatementConverters/ClauseProcessors/UpdateClauseConverter.cs @@ -1,10 +1,10 @@ -using TSQL.Clauses; +using System.Collections.Generic; namespace EmbeddedSQLTester.SQLitePlatformConversion.StatementConverters.ClauseProcessors { internal class UpdateClauseConverter : ClauseConverterBase { - public UpdateClauseConverter(TSQLClause clauseParameter) : base(clauseParameter) + public UpdateClauseConverter(List tokens) : base(tokens) { } diff --git a/EmbeddedSQLTester/SQLitePlatformConversion/StatementConverters/ClauseProcessors/WhereClauseConverter.cs b/EmbeddedSQLTester/SQLitePlatformConversion/StatementConverters/ClauseProcessors/WhereClauseConverter.cs index 915e7d7..ad51755 100644 --- a/EmbeddedSQLTester/SQLitePlatformConversion/StatementConverters/ClauseProcessors/WhereClauseConverter.cs +++ b/EmbeddedSQLTester/SQLitePlatformConversion/StatementConverters/ClauseProcessors/WhereClauseConverter.cs @@ -1,11 +1,11 @@ +using System.Collections.Generic; using System.Text; -using TSQL.Clauses; namespace EmbeddedSQLTester.SQLitePlatformConversion.StatementConverters.ClauseProcessors { internal class WhereClauseConverter : ClauseConverterBase { - public WhereClauseConverter(TSQLClause clause) : base(clause) + public WhereClauseConverter(List tokens) : base(tokens) { } diff --git a/EmbeddedSQLTester/SQLitePlatformConversion/StatementConverters/DeleteStatementConverter.cs b/EmbeddedSQLTester/SQLitePlatformConversion/StatementConverters/DeleteStatementConverter.cs index 50d08eb..eaec43b 100644 --- a/EmbeddedSQLTester/SQLitePlatformConversion/StatementConverters/DeleteStatementConverter.cs +++ b/EmbeddedSQLTester/SQLitePlatformConversion/StatementConverters/DeleteStatementConverter.cs @@ -1,19 +1,87 @@ using System.Collections.Generic; using EmbeddedSQLTester.SQLitePlatformConversion.StatementConverters.ClauseProcessors; -using TSQL.Statements; +using Microsoft.SqlServer.TransactSql.ScriptDom; namespace EmbeddedSQLTester.SQLitePlatformConversion.StatementConverters { internal class DeleteStatementConverter { - public List GetClauseProcessorList(TSQLDeleteStatement statement) + public List GetClauseProcessorList(DeleteStatement statement) { + var spec = statement.DeleteSpecification; + + // DELETE keyword tokens (just "DELETE") + var deleteTokens = ExtractDeleteKeywordTokens(statement); + var fromTokens = ExtractFromTokens(spec); + var whereTokens = SqlToken.ExtractTokens(spec.WhereClause); + return new List { - new GeneralClauseConverter(statement.Delete), - new FromClauseConverter(statement.From), - new WhereClauseConverter(statement.Where), + new GeneralClauseConverter(deleteTokens), + new FromClauseConverter(fromTokens), + new WhereClauseConverter(whereTokens), }; } + + private List ExtractDeleteKeywordTokens(DeleteStatement statement) + { + var tokens = new List(); + var stream = statement.ScriptTokenStream; + + // Just the DELETE keyword + for (int i = statement.FirstTokenIndex; i <= statement.LastTokenIndex; i++) + { + var t = stream[i]; + if (t.TokenType == TSqlTokenType.WhiteSpace) continue; + if (t.TokenType == TSqlTokenType.Delete) + { + tokens.Add(new SqlToken + { + Text = t.Text, + BeginPosition = t.Offset, + EndPosition = t.Offset + t.Text.Length - 1 + }); + break; + } + } + + return tokens; + } + + private List ExtractFromTokens(DeleteSpecification spec) + { + // For DELETE FROM, the FromClause on DeleteSpecification may be null + // but the target has the table. We need FROM + target. + var tokens = new List(); + var stream = spec.ScriptTokenStream; + + // Find FROM keyword and everything up to WHERE (or end) + int startIndex = spec.Target.FirstTokenIndex; + int endIndex = spec.WhereClause != null ? spec.WhereClause.FirstTokenIndex - 1 : spec.LastTokenIndex; + + // Look for FROM keyword before the target + for (int i = startIndex - 1; i >= spec.FirstTokenIndex; i--) + { + if (stream[i].TokenType == TSqlTokenType.From) + { + startIndex = i; + break; + } + } + + for (int i = startIndex; i <= endIndex; i++) + { + var t = stream[i]; + if (t.TokenType == TSqlTokenType.WhiteSpace) continue; + tokens.Add(new SqlToken + { + Text = t.Text, + BeginPosition = t.Offset, + EndPosition = t.Offset + t.Text.Length - 1 + }); + } + + return tokens; + } } } \ No newline at end of file diff --git a/EmbeddedSQLTester/SQLitePlatformConversion/StatementConverters/ExpressionProcessors/ExpressionConverterBase.cs b/EmbeddedSQLTester/SQLitePlatformConversion/StatementConverters/ExpressionProcessors/ExpressionConverterBase.cs index 52d98fb..92945fd 100644 --- a/EmbeddedSQLTester/SQLitePlatformConversion/StatementConverters/ExpressionProcessors/ExpressionConverterBase.cs +++ b/EmbeddedSQLTester/SQLitePlatformConversion/StatementConverters/ExpressionProcessors/ExpressionConverterBase.cs @@ -1,30 +1,27 @@ using System.Collections.Generic; using System.Text; using EmbeddedSQLTester.SQLitePlatformConversion.StatementConverters.ClauseProcessors; -using TSQL.Elements; -using TSQL.Expressions; -using TSQL.Tokens; namespace EmbeddedSQLTester.SQLitePlatformConversion.StatementConverters.ExpressionProcessors { internal abstract class ExpressionConverterBase : ConverterBase { - private readonly TSQLValues _expression; + private readonly List _tokens; private StringBuilder _stringBuilder; - protected List Tokens; + protected List Tokens; - protected ExpressionConverterBase(TSQLValues expression) + protected ExpressionConverterBase(List tokens) { - _expression = expression; + _tokens = tokens; } public override string Convert() { _stringBuilder = new StringBuilder(); - if (_expression != null) + if (_tokens != null) { - Tokens = _expression.Tokens; + Tokens = _tokens; ConvertHelper(); } diff --git a/EmbeddedSQLTester/SQLitePlatformConversion/StatementConverters/ExpressionProcessors/ValuesExpressionConverter.cs b/EmbeddedSQLTester/SQLitePlatformConversion/StatementConverters/ExpressionProcessors/ValuesExpressionConverter.cs index 6b22446..8e607da 100644 --- a/EmbeddedSQLTester/SQLitePlatformConversion/StatementConverters/ExpressionProcessors/ValuesExpressionConverter.cs +++ b/EmbeddedSQLTester/SQLitePlatformConversion/StatementConverters/ExpressionProcessors/ValuesExpressionConverter.cs @@ -1,12 +1,11 @@ -using EmbeddedSQLTester.SQLitePlatformConversion.StatementConverters.ClauseProcessors; -using TSQL.Elements; -using TSQL.Expressions; +using System.Collections.Generic; +using EmbeddedSQLTester.SQLitePlatformConversion.StatementConverters.ClauseProcessors; namespace EmbeddedSQLTester.SQLitePlatformConversion.StatementConverters.ExpressionProcessors { internal class ValuesExpressionConverter : ExpressionConverterBase { - public ValuesExpressionConverter(TSQLValues clause) : base(clause) + public ValuesExpressionConverter(List tokens) : base(tokens) { } diff --git a/EmbeddedSQLTester/SQLitePlatformConversion/StatementConverters/InsertStatementConverter.cs b/EmbeddedSQLTester/SQLitePlatformConversion/StatementConverters/InsertStatementConverter.cs index 60f426b..6dae12b 100644 --- a/EmbeddedSQLTester/SQLitePlatformConversion/StatementConverters/InsertStatementConverter.cs +++ b/EmbeddedSQLTester/SQLitePlatformConversion/StatementConverters/InsertStatementConverter.cs @@ -1,19 +1,51 @@ using System.Collections.Generic; using EmbeddedSQLTester.SQLitePlatformConversion.StatementConverters.ClauseProcessors; using EmbeddedSQLTester.SQLitePlatformConversion.StatementConverters.ExpressionProcessors; -using TSQL.Statements; +using Microsoft.SqlServer.TransactSql.ScriptDom; namespace EmbeddedSQLTester.SQLitePlatformConversion.StatementConverters { internal class InsertStatementConverter { - public List GetClauseProcessorList(TSQLInsertStatement statement) + public List GetClauseProcessorList(InsertStatement statement) { + var spec = statement.InsertSpecification; + + // INSERT clause: from INSERT keyword through columns + var insertTokens = ExtractInsertClauseTokens(statement); + + // VALUES clause + var valuesTokens = SqlToken.ExtractTokens(spec.InsertSource); + return new List { - new InsertClauseConverter(statement.Insert), - new ValuesExpressionConverter(statement.Values) + new InsertClauseConverter(insertTokens), + new ValuesExpressionConverter(valuesTokens) }; } + + private List ExtractInsertClauseTokens(InsertStatement statement) + { + var spec = statement.InsertSpecification; + var tokens = new List(); + var stream = statement.ScriptTokenStream; + + // From INSERT to just before VALUES (the InsertSource) + int endIndex = spec.InsertSource.FirstTokenIndex; + + for (int i = statement.FirstTokenIndex; i < endIndex; i++) + { + var t = stream[i]; + if (t.TokenType == TSqlTokenType.WhiteSpace) continue; + tokens.Add(new SqlToken + { + Text = t.Text, + BeginPosition = t.Offset, + EndPosition = t.Offset + t.Text.Length - 1 + }); + } + + return tokens; + } } } \ No newline at end of file diff --git a/EmbeddedSQLTester/SQLitePlatformConversion/StatementConverters/LimitStatementConverter.cs b/EmbeddedSQLTester/SQLitePlatformConversion/StatementConverters/LimitStatementConverter.cs index b014b54..344ed89 100644 --- a/EmbeddedSQLTester/SQLitePlatformConversion/StatementConverters/LimitStatementConverter.cs +++ b/EmbeddedSQLTester/SQLitePlatformConversion/StatementConverters/LimitStatementConverter.cs @@ -1,11 +1,11 @@ -using EmbeddedSQLTester.SQLitePlatformConversion.StatementConverters.ClauseProcessors; -using TSQL.Clauses; +using System.Collections.Generic; +using EmbeddedSQLTester.SQLitePlatformConversion.StatementConverters.ClauseProcessors; namespace EmbeddedSQLTester.SQLitePlatformConversion.StatementConverters { internal class LimitStatementConverter : ClauseConverterBase { - public LimitStatementConverter(TSQLClause clauseParameter) : base(clauseParameter) + public LimitStatementConverter(List tokens) : base(tokens) { } diff --git a/EmbeddedSQLTester/SQLitePlatformConversion/StatementConverters/SelectStatementConverter.cs b/EmbeddedSQLTester/SQLitePlatformConversion/StatementConverters/SelectStatementConverter.cs index 5d8c72c..20ff1ea 100644 --- a/EmbeddedSQLTester/SQLitePlatformConversion/StatementConverters/SelectStatementConverter.cs +++ b/EmbeddedSQLTester/SQLitePlatformConversion/StatementConverters/SelectStatementConverter.cs @@ -1,22 +1,80 @@ using System.Collections.Generic; using EmbeddedSQLTester.SQLitePlatformConversion.StatementConverters.ClauseProcessors; -using TSQL.Statements; +using Microsoft.SqlServer.TransactSql.ScriptDom; namespace EmbeddedSQLTester.SQLitePlatformConversion.StatementConverters { internal class SelectStatementConverter { - public List GetClauseProcessorList(TSQLSelectStatement statement) + public List GetClauseProcessorList(SelectStatement statement) { + var querySpec = (QuerySpecification)statement.QueryExpression; + + var selectTokens = SqlToken.ExtractTokens(querySpec); + var fromTokens = SqlToken.ExtractTokens(querySpec.FromClause); + var whereTokens = SqlToken.ExtractTokens(querySpec.WhereClause); + var groupByTokens = SqlToken.ExtractTokens(querySpec.GroupByClause); + + // ORDER BY tokens must include OFFSET/FETCH if present + var orderByTokens = ExtractOrderByWithOffset(querySpec); + + // Remove FROM and beyond from select tokens to isolate just the SELECT clause + if (querySpec.FromClause != null) + { + var selectOnly = new List(); + var fromStart = querySpec.FromClause.FirstTokenIndex; + for (int i = querySpec.FirstTokenIndex; i <= querySpec.LastTokenIndex; i++) + { + if (i >= fromStart) break; + var t = querySpec.ScriptTokenStream[i]; + if (t.TokenType == TSqlTokenType.WhiteSpace) continue; + selectOnly.Add(new SqlToken + { + Text = t.Text, + BeginPosition = t.Offset, + EndPosition = t.Offset + t.Text.Length - 1 + }); + } + selectTokens = selectOnly; + } + return new List { - new GeneralClauseConverter(statement.Select), - new FromClauseConverter(statement.From), - new WhereClauseConverter(statement.Where), - new GeneralClauseConverter(statement.GroupBy), - new OrderByClauseConverter(statement.OrderBy), - new LimitStatementConverter(statement.Select) + new GeneralClauseConverter(selectTokens), + new FromClauseConverter(fromTokens), + new WhereClauseConverter(whereTokens), + new GeneralClauseConverter(groupByTokens), + new OrderByClauseConverter(orderByTokens), + new LimitStatementConverter(selectTokens) }; } + + private List ExtractOrderByWithOffset(QuerySpecification querySpec) + { + if (querySpec.OrderByClause == null) return null; + + // Determine end index: include OffsetClause if present + int startIndex = querySpec.OrderByClause.FirstTokenIndex; + int endIndex = querySpec.OrderByClause.LastTokenIndex; + + if (querySpec.OffsetClause != null) + { + endIndex = querySpec.OffsetClause.LastTokenIndex; + } + + var tokens = new List(); + for (int i = startIndex; i <= endIndex; i++) + { + var t = querySpec.ScriptTokenStream[i]; + if (t.TokenType == TSqlTokenType.WhiteSpace) continue; + tokens.Add(new SqlToken + { + Text = t.Text, + BeginPosition = t.Offset, + EndPosition = t.Offset + t.Text.Length - 1 + }); + } + return tokens; + } } } \ No newline at end of file diff --git a/EmbeddedSQLTester/SQLitePlatformConversion/StatementConverters/SqlToken.cs b/EmbeddedSQLTester/SQLitePlatformConversion/StatementConverters/SqlToken.cs new file mode 100644 index 0000000..9a31f52 --- /dev/null +++ b/EmbeddedSQLTester/SQLitePlatformConversion/StatementConverters/SqlToken.cs @@ -0,0 +1,33 @@ +using System.Collections.Generic; +using Microsoft.SqlServer.TransactSql.ScriptDom; + +namespace EmbeddedSQLTester.SQLitePlatformConversion.StatementConverters +{ + internal class SqlToken + { + public string Text { get; set; } + public int BeginPosition { get; set; } + public int EndPosition { get; set; } + + public static List ExtractTokens(TSqlFragment fragment) + { + if (fragment == null) return null; + + var tokens = new List(); + for (int i = fragment.FirstTokenIndex; i <= fragment.LastTokenIndex; i++) + { + var t = fragment.ScriptTokenStream[i]; + if (t.TokenType == TSqlTokenType.WhiteSpace) continue; + tokens.Add(new SqlToken + { + Text = t.Text, + BeginPosition = t.Offset, + EndPosition = t.Offset + t.Text.Length - 1 + }); + } + return tokens; + } + } +} + + diff --git a/EmbeddedSQLTester/SQLitePlatformConversion/StatementConverters/TokenBasedStatementParser.cs b/EmbeddedSQLTester/SQLitePlatformConversion/StatementConverters/TokenBasedStatementParser.cs new file mode 100644 index 0000000..ddcf400 --- /dev/null +++ b/EmbeddedSQLTester/SQLitePlatformConversion/StatementConverters/TokenBasedStatementParser.cs @@ -0,0 +1,183 @@ +using System.Collections.Generic; +using System.Linq; +using Microsoft.SqlServer.TransactSql.ScriptDom; + +namespace EmbeddedSQLTester.SQLitePlatformConversion.StatementConverters +{ + /// + /// Fallback parser that splits a raw ScriptDom token stream into clause-level + /// token lists by scanning for keyword boundaries. Used when the AST parser + /// fails (e.g., UPDATE with table alias, trailing unmatched parenthesis). + /// + internal static class TokenBasedStatementParser + { + public static string DetectStatementType(IList stream) + { + for (int i = 0; i < stream.Count; i++) + { + if (stream[i].TokenType == TSqlTokenType.WhiteSpace) continue; + switch (stream[i].TokenType) + { + case TSqlTokenType.Select: return "SELECT"; + case TSqlTokenType.Update: return "UPDATE"; + case TSqlTokenType.Insert: return "INSERT"; + case TSqlTokenType.Delete: return "DELETE"; + } + break; + } + return null; + } + + public static Dictionary> SplitIntoClauses( + IList stream, string statementType) + { + switch (statementType) + { + case "SELECT": return SplitSelectClauses(stream); + case "UPDATE": return SplitUpdateClauses(stream); + case "INSERT": return SplitInsertClauses(stream); + case "DELETE": return SplitDeleteClauses(stream); + default: return new Dictionary>(); + } + } + + private static Dictionary> SplitSelectClauses(IList stream) + { + var result = new Dictionary>(); + var boundaries = FindKeywordBoundaries(stream, new[] + { + TSqlTokenType.Select, TSqlTokenType.From, TSqlTokenType.Where, + TSqlTokenType.Order, TSqlTokenType.Group + }); + + AssignClause(result, "SELECT", stream, boundaries, TSqlTokenType.Select); + AssignClause(result, "FROM", stream, boundaries, TSqlTokenType.From); + AssignClause(result, "WHERE", stream, boundaries, TSqlTokenType.Where); + AssignClause(result, "GROUPBY", stream, boundaries, TSqlTokenType.Group); + AssignClause(result, "ORDERBY", stream, boundaries, TSqlTokenType.Order); + + return result; + } + + private static Dictionary> SplitUpdateClauses(IList stream) + { + var result = new Dictionary>(); + var boundaries = FindKeywordBoundaries(stream, new[] + { + TSqlTokenType.Update, TSqlTokenType.Set, TSqlTokenType.From, TSqlTokenType.Where + }); + + AssignClause(result, "UPDATE", stream, boundaries, TSqlTokenType.Update); + AssignClause(result, "SET", stream, boundaries, TSqlTokenType.Set); + AssignClause(result, "FROM", stream, boundaries, TSqlTokenType.From); + AssignClause(result, "WHERE", stream, boundaries, TSqlTokenType.Where); + + return result; + } + + private static Dictionary> SplitInsertClauses(IList stream) + { + var result = new Dictionary>(); + + // Find VALUES keyword position + int valuesIdx = -1; + for (int i = 0; i < stream.Count; i++) + { + if (stream[i].TokenType == TSqlTokenType.WhiteSpace) continue; + if (stream[i].Text.Equals("VALUES", System.StringComparison.OrdinalIgnoreCase)) + { + valuesIdx = i; + break; + } + } + + if (valuesIdx >= 0) + { + result["INSERT"] = ExtractTokenRange(stream, 0, valuesIdx - 1); + result["VALUES"] = ExtractTokenRange(stream, valuesIdx, stream.Count - 1); + } + + return result; + } + + private static Dictionary> SplitDeleteClauses(IList stream) + { + var result = new Dictionary>(); + var boundaries = FindKeywordBoundaries(stream, new[] + { + TSqlTokenType.Delete, TSqlTokenType.From, TSqlTokenType.Where + }); + + // DELETE keyword only + if (boundaries.ContainsKey(TSqlTokenType.Delete)) + { + int deleteStart = boundaries[TSqlTokenType.Delete]; + result["DELETE"] = ExtractTokenRange(stream, deleteStart, deleteStart); + } + + AssignClause(result, "FROM", stream, boundaries, TSqlTokenType.From); + AssignClause(result, "WHERE", stream, boundaries, TSqlTokenType.Where); + + return result; + } + + private static Dictionary FindKeywordBoundaries( + IList stream, TSqlTokenType[] keywords) + { + var boundaries = new Dictionary(); + for (int i = 0; i < stream.Count; i++) + { + var tokenType = stream[i].TokenType; + if (tokenType == TSqlTokenType.EndOfFile) break; + if (keywords.Contains(tokenType) && !boundaries.ContainsKey(tokenType)) + { + boundaries[tokenType] = i; + } + } + return boundaries; + } + + private static void AssignClause(Dictionary> result, string clauseName, + IList stream, Dictionary boundaries, TSqlTokenType keyword) + { + if (!boundaries.ContainsKey(keyword)) return; + + int start = boundaries[keyword]; + + // Find the next clause boundary after this one + int end = stream.Count - 1; + // Remove EndOfFile token + if (stream[end].TokenType == TSqlTokenType.EndOfFile) end--; + + // Sort boundaries by position and find the next one after 'start' + foreach (var kvp in boundaries) + { + if (kvp.Value > start && kvp.Value < end) + end = kvp.Value - 1; + } + + result[clauseName] = ExtractTokenRange(stream, start, end); + } + + private static List ExtractTokenRange(IList stream, int start, int end) + { + var tokens = new List(); + for (int i = start; i <= end && i < stream.Count; i++) + { + var t = stream[i]; + if (t.TokenType == TSqlTokenType.WhiteSpace) continue; + if (t.TokenType == TSqlTokenType.EndOfFile) continue; + // Skip trailing unmatched parentheses + if (t.TokenType == TSqlTokenType.RightParenthesis && i == end) continue; + tokens.Add(new SqlToken + { + Text = t.Text, + BeginPosition = t.Offset, + EndPosition = t.Offset + t.Text.Length - 1 + }); + } + return tokens.Count > 0 ? tokens : null; + } + } +} + diff --git a/EmbeddedSQLTester/SQLitePlatformConversion/StatementConverters/UpdateStatementConverter.cs b/EmbeddedSQLTester/SQLitePlatformConversion/StatementConverters/UpdateStatementConverter.cs index 511a03b..a5be6e1 100644 --- a/EmbeddedSQLTester/SQLitePlatformConversion/StatementConverters/UpdateStatementConverter.cs +++ b/EmbeddedSQLTester/SQLitePlatformConversion/StatementConverters/UpdateStatementConverter.cs @@ -1,20 +1,100 @@ using System.Collections.Generic; using EmbeddedSQLTester.SQLitePlatformConversion.StatementConverters.ClauseProcessors; -using TSQL.Statements; +using Microsoft.SqlServer.TransactSql.ScriptDom; namespace EmbeddedSQLTester.SQLitePlatformConversion.StatementConverters { internal class UpdateStatementConverter { - public List GetClauseProcessorList(TSQLUpdateStatement statement) + public List GetClauseProcessorList(UpdateStatement statement) { + var spec = statement.UpdateSpecification; + + // UPDATE clause: from UPDATE keyword to the target + var updateTokens = ExtractUpdateClauseTokens(statement); + var setTokens = ExtractSetClauseTokens(spec); + var fromTokens = SqlToken.ExtractTokens(spec.FromClause); + var whereTokens = SqlToken.ExtractTokens(spec.WhereClause); + return new List { - new UpdateClauseConverter(statement.Update), - new GeneralClauseConverter(statement.Set), - new FromClauseConverter(statement.From), - new WhereClauseConverter(statement.Where), + new UpdateClauseConverter(updateTokens), + new GeneralClauseConverter(setTokens), + new FromClauseConverter(fromTokens), + new WhereClauseConverter(whereTokens), }; } + + private List ExtractUpdateClauseTokens(UpdateStatement statement) + { + var spec = statement.UpdateSpecification; + // Tokens from "UPDATE" to end of target (before SET) + var tokens = new List(); + int endIndex = spec.SetClauses[0].FirstTokenIndex; + // Walk backward from SET to skip whitespace + for (int i = statement.FirstTokenIndex; i < endIndex; i++) + { + var t = statement.ScriptTokenStream[i]; + if (t.TokenType == TSqlTokenType.WhiteSpace) continue; + // Stop if we hit SET keyword + if (t.TokenType == TSqlTokenType.Set) break; + tokens.Add(new SqlToken + { + Text = t.Text, + BeginPosition = t.Offset, + EndPosition = t.Offset + t.Text.Length - 1 + }); + } + return tokens; + } + + private List ExtractSetClauseTokens(UpdateSpecification spec) + { + if (spec.SetClauses == null || spec.SetClauses.Count == 0) return null; + + var tokens = new List(); + // Find the SET keyword token (it's just before the first SetClause) + var stream = spec.ScriptTokenStream; + var firstSetClauseIdx = spec.SetClauses[0].FirstTokenIndex; + + // Search backwards for SET keyword + for (int i = firstSetClauseIdx - 1; i >= 0; i--) + { + if (stream[i].TokenType == TSqlTokenType.Set) + { + tokens.Add(new SqlToken + { + Text = stream[i].Text, + BeginPosition = stream[i].Offset, + EndPosition = stream[i].Offset + stream[i].Text.Length - 1 + }); + break; + } + } + + // Determine end index: before FROM or WHERE or end of spec + int endIndex = spec.LastTokenIndex; + if (spec.FromClause != null) + endIndex = spec.FromClause.FirstTokenIndex - 1; + else if (spec.WhereClause != null) + endIndex = spec.WhereClause.FirstTokenIndex - 1; + + var lastSetClause = spec.SetClauses[spec.SetClauses.Count - 1]; + endIndex = lastSetClause.LastTokenIndex; + + for (int i = spec.SetClauses[0].FirstTokenIndex; i <= endIndex; i++) + { + var t = stream[i]; + if (t.TokenType == TSqlTokenType.WhiteSpace) continue; + tokens.Add(new SqlToken + { + Text = t.Text, + BeginPosition = t.Offset, + EndPosition = t.Offset + t.Text.Length - 1 + }); + } + + return tokens; + } } } \ No newline at end of file diff --git a/IMPLEMENTATION_PLAN.md b/IMPLEMENTATION_PLAN.md new file mode 100644 index 0000000..7e60319 --- /dev/null +++ b/IMPLEMENTATION_PLAN.md @@ -0,0 +1,298 @@ +# Implementation Plan: Migrate from TSQL.Parser to Microsoft.SqlServer.TransactSql.ScriptDom + +## Overview + +Replace the `TSQL.Parser` (v2.6.0) NuGet dependency with `Microsoft.SqlServer.TransactSql.ScriptDom` for T-SQL parsing. The existing unit tests remain unchanged and serve as proof of correctness. + +--- + +## 1. Key Differences Between Libraries + +| Aspect | TSQL.Parser | ScriptDom | +| ----------------------- | ------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Parsing entry point** | `TSQLStatementReader.ParseStatements(sql)` → `List` | `TSqlParser.Parse(reader, out errors)` → `TSqlFragment` (AST root) | +| **Statement types** | `TSQLSelectStatement`, `TSQLUpdateStatement`, etc. | `SelectStatement`, `UpdateStatement`, etc. (in `Microsoft.SqlServer.TransactSql.ScriptDom`) | +| **Clause access** | `statement.Select`, `statement.From`, `statement.Where` — each returns a `TSQLClause` with a flat `List` | Rich nested AST objects (`QuerySpecification.FromClause`, `.WhereClause`, etc.) — tree structure, not flat tokens | +| **Token model** | `TSQLToken` with `.Text`, `.BeginPosition`, `.EndPosition` | `TSqlParserToken` with `.Text`, `.Offset`, `.TokenType` accessible via `ScriptTokenStream` | +| **Traversal** | Manual iteration over flat token lists per clause | Visitor pattern (`TSqlFragmentVisitor`) or direct AST property navigation; each fragment also carries `ScriptTokenStream` + `FirstTokenIndex`/`LastTokenIndex` | +| **SQL regeneration** | Manual token-by-token string building (current approach) | `Sql150ScriptGenerator.GenerateScript()` for AST-to-SQL, or manual token manipulation | + +## 2. Refactoring Strategy + +### Recommended approach: **Token-stream-based conversion preserving the current architecture** + +The current code processes clauses as flat token streams and applies text transformations token-by-token. ScriptDom also exposes a token stream on every AST fragment. This means we can **preserve the existing converter architecture** (ConverterBase → ClauseConverterBase → specific converters) while swapping out the underlying token type. + +This minimizes risk and keeps the refactoring scope contained. + +### Alternative considered (not recommended): Full AST visitor rewrite + +A complete rewrite using `TSqlFragmentVisitor` to walk the AST and produce transformed SQL would be more idiomatic for ScriptDom but would require fundamentally redesigning every converter class—far more effort and risk for the same result. + +--- + +## 3. Step-by-Step Plan + +### Step 1: Update NuGet Dependencies + +**File:** `EmbeddedSQLTester/EmbeddedSQLTester.csproj` + +- Remove: `` +- Add: `` +- Note: Also drop `netcoreapp3.1` and `netstandard2.1` targets if desired—ScriptDom supports `netstandard2.0+` and `net6.0+`. At minimum, keep `net6.0` and `net7.0`. +- Update the test project's `TargetFrameworks` similarly if needed. + +--- + +### Step 2: Create a Lightweight Token Wrapper + +**New file:** `EmbeddedSQLTester/SQLitePlatformConversion/StatementConverters/SqlToken.cs` + +Create a simple class that normalizes ScriptDom's `TSqlParserToken` into the shape the existing converters expect: + +```csharp +internal class SqlToken +{ + public string Text { get; set; } + public int BeginPosition { get; set; } + public int EndPosition { get; set; } +} +``` + +Add a static helper to extract a `List` from a `TSqlFragment`'s token range (`FirstTokenIndex` to `LastTokenIndex` on the `ScriptTokenStream`). + +--- + +### Step 3: Refactor `ClauseConverterBase` + +**File:** `EmbeddedSQLTester/SQLitePlatformConversion/StatementConverters/ClauseProcessors/ClauseConverterBase.cs` + +- Change the constructor to accept a `List` (or the fragment directly) instead of `TSQLClause`. +- Change `protected List Tokens` → `protected List Tokens`. +- Remove `using TSQL.Clauses;` and `using TSQL.Tokens;`. +- The `Convert()` and `ConvertHelper()` methods remain structurally the same since they iterate `Tokens[i].Text`, `.BeginPosition`, `.EndPosition`—the same properties exist on our wrapper. + +--- + +### Step 4: Refactor `ExpressionConverterBase` + +**File:** `EmbeddedSQLTester/SQLitePlatformConversion/StatementConverters/ExpressionProcessors/ExpressionConverterBase.cs` + +- Same approach as ClauseConverterBase: accept `List` instead of `TSQLValues`. +- Remove `using TSQL.Elements;`, `using TSQL.Expressions;`, `using TSQL.Tokens;`. + +--- + +### Step 5: Update All Clause/Expression Converters + +Each converter's constructor signature changes from accepting `TSQLClause` to `List`. No logic changes needed since they all operate on `Tokens[position].Text` which remains the same. + +| File | Change | +| --------------------------------------------------- | ------------------------------------------------------------------------------------------------------- | +| `ClauseProcessors/GeneralClauseConverter.cs` | Constructor: `TSQLClause` → `List`. Remove `using TSQL.Clauses;`. | +| `ClauseProcessors/FromClauseConverter.cs` | Constructor: `TSQLClause` → `List`. Remove `using TSQL.Clauses;`. | +| `ClauseProcessors/WhereClauseConverter.cs` | Constructor: `TSQLClause` → `List`. Remove `using TSQL.Clauses;`. | +| `ClauseProcessors/OrderByClauseConverter.cs` | Constructor: `TSQLClause` → `List`. Remove `using TSQL.Clauses;`. | +| `ClauseProcessors/InsertClauseConverter.cs` | Constructor: `TSQLClause` → `List`. Remove `using TSQL.Clauses;`. | +| `ClauseProcessors/UpdateClauseConverter.cs` | Constructor: `TSQLClause` → `List`. Remove `using TSQL.Clauses;`. | +| `StatementConverters/LimitStatementConverter.cs` | Constructor: `TSQLClause` → `List`. Remove `using TSQL.Clauses;`. | +| `ExpressionProcessors/ValuesExpressionConverter.cs` | Constructor: `TSQLValues` → `List`. Remove `using TSQL.Elements;`, `using TSQL.Expressions;`. | + +--- + +### Step 6: Refactor Statement Converters to Extract Tokens from ScriptDom AST + +Each statement converter currently receives a TSQL.Parser statement and pulls its clause properties. Now they must receive a ScriptDom statement, navigate the AST to find the relevant fragments, and extract `List` for each clause. + +#### 6a. `SelectStatementConverter.cs` + +**Current:** Receives `TSQLSelectStatement`, accesses `.Select`, `.From`, `.Where`, `.GroupBy`, `.OrderBy`. + +**New:** Receives ScriptDom's `SelectStatement`. Navigate: + +- `selectStatement.QueryExpression` → cast to `QuerySpecification` +- SELECT tokens: `querySpec` itself from `SELECT` keyword to before `FROM` (use `querySpec.SelectElements` token range) +- FROM tokens: `querySpec.FromClause` fragment +- WHERE tokens: `querySpec.WhereClause` fragment +- GROUP BY tokens: `querySpec.GroupByClause` fragment +- ORDER BY tokens: `selectStatement.OrderByClause` fragment (note: on `SelectStatement`, not `QuerySpecification`) +- TOP/LIMIT: `querySpec.TopRowFilter` fragment + +Each fragment has `FirstTokenIndex` / `LastTokenIndex` on the `ScriptTokenStream`, use those to build `List`. + +#### 6b. `UpdateStatementConverter.cs` + +**Current:** Receives `TSQLUpdateStatement`, accesses `.Update`, `.Set`, `.From`, `.Where`. + +**New:** Receives ScriptDom's `UpdateStatement`. Navigate: + +- `updateStatement.UpdateSpecification.Target` for the UPDATE clause +- `updateStatement.UpdateSpecification.SetClauses` for SET +- `updateStatement.UpdateSpecification.FromClause` for FROM +- `updateStatement.UpdateSpecification.WhereClause` for WHERE + +Build `List` from each fragment's token range, prepending the keyword token (e.g., "UPDATE", "SET"). + +#### 6c. `InsertStatementConverter.cs` + +**Current:** Receives `TSQLInsertStatement`, accesses `.Insert`, `.Values`. + +**New:** Receives ScriptDom's `InsertStatement`. Navigate: + +- `insertStatement.InsertSpecification.Target` + `.Columns` for the INSERT clause +- `insertStatement.InsertSpecification.InsertSource` (cast to `ValuesInsertSource`) for VALUES + +#### 6d. `DeleteStatementConverter.cs` + +**Current:** Receives `TSQLDeleteStatement`, accesses `.Delete`, `.From`, `.Where`. + +**New:** Receives ScriptDom's `DeleteStatement`. Navigate: + +- `deleteStatement.DeleteSpecification` for the DELETE keyword +- `deleteStatement.DeleteSpecification.Target` + `FromClause` for FROM +- `deleteStatement.DeleteSpecification.WhereClause` for WHERE + +--- + +### Step 7: Refactor `SQLServerToOrmliteSQLiteDialectConverter` + +**File:** `EmbeddedSQLTester/SQLitePlatformConversion/SQLServerToOrmliteSQLiteDialectConverter.cs` + +Replace the parsing logic: + +```csharp +// OLD +using TSQL; +using TSQL.Statements; +var statements = TSQLStatementReader.ParseStatements(sqlInput); +var statement = statements[0]; +if (statement.GetType() == typeof(TSQLSelectStatement)) { ... } + +// NEW +using Microsoft.SqlServer.TransactSql.ScriptDom; +var parser = new TSql150Parser(false); +using var reader = new StringReader(sqlInput); +var fragment = parser.Parse(reader, out var errors); +var batch = ((TSqlScript)fragment).Batches[0]; +var statement = batch.Statements[0]; +if (statement is SelectStatement selectStatement) { ... } +else if (statement is UpdateStatement updateStatement) { ... } +else if (statement is InsertStatement insertStatement) { ... } +else if (statement is DeleteStatement deleteStatement) { ... } +``` + +--- + +### Step 8: Add a Token Extraction Helper + +**New file or method in:** `SqlToken.cs` or a new `TokenExtractor.cs` + +Create a utility that converts a `TSqlFragment` into `List`: + +```csharp +internal static List ExtractTokens(TSqlFragment fragment) +{ + if (fragment == null) return null; + + var tokens = new List(); + for (int i = fragment.FirstTokenIndex; i <= fragment.LastTokenIndex; i++) + { + var t = fragment.ScriptTokenStream[i]; + if (t.TokenType == TSqlTokenType.WhiteSpace) continue; // skip whitespace tokens + tokens.Add(new SqlToken + { + Text = t.Text, + BeginPosition = t.Offset, + EndPosition = t.Offset + t.Text.Length + }); + } + return tokens; +} +``` + +This bridges the gap between ScriptDom's AST and the existing token-iteration logic. + +--- + +### Step 9: Handle Edge Cases & Special Token Mappings + +Several things to verify/adjust during implementation: + +1. **Token boundaries (BeginPosition/EndPosition spacing):** The current `ClauseConverterBase.ConvertHelper()` uses `Tokens[i].BeginPosition - lastEndPosition > 1` to decide whether to insert a space. ScriptDom token offsets are absolute to the full input string. The wrapper must either preserve these absolute offsets or adjust the spacing logic to simply always insert a single space between non-empty tokens. + +2. **Null clauses:** The current code passes `null` clauses (e.g., a SELECT with no WHERE). The base classes already handle `null` gracefully—the wrapper extraction must also return `null` for absent clauses. + +3. **Token ordering in UPDATE SET:** TSQL.Parser gives the entire `SET` clause as a flat token list including the `SET` keyword. ScriptDom's `SetClauses` is a collection of `SetClause` objects. The token extraction must span from the `SET` keyword token through all set clauses. + +4. **Token ordering for INSERT:** TSQL.Parser includes `INSERT INTO tablename (col1, col2)` as one clause. ScriptDom splits this into `Target` and `Columns`. The statement converter must concatenate: `["INSERT", "INTO"] + target tokens + column tokens`. + +5. **VALUES token list:** ScriptDom uses `ValuesInsertSource` containing `RowValues`. Extract the full token range from the `ValuesInsertSource` fragment. + +6. **TOP clause handling:** In TSQL.Parser, `TOP 10` tokens live inside the SELECT clause token list. In ScriptDom, `TopRowFilter` is a separate fragment on `QuerySpecification`. The `LimitStatementConverter` currently scans the SELECT tokens for "top". After refactoring, either: + - Include `TopRowFilter` tokens in the select token list (natural if extracting from `QuerySpecification` full range), OR + - Pass `TopRowFilter` tokens separately to `LimitStatementConverter`. + +7. **Closing parenthesis in LIKE:** Test `Convert_LikeStatementsAreConverted` input has a trailing `)` that TSQL.Parser apparently drops. Verify ScriptDom behavior matches. + +--- + +### Step 10: Verify with Existing Unit Tests + +Run all existing tests without modification: + +```bash +dotnet test EmbeddedSQLTester.Tests/ +``` + +All 17 tests must pass: + +- `SQLitePlatformConverterTests` (13 select tests) +- `SQLitePlatformConverterUpdateTests` (3 update tests) +- `SQLitePlatformConverterInsertTests` (1 insert test) +- `SQLitePlatformConverterDeleteTests` (1 delete test) +- `MyExtensionTests` (1 extension method test) + +--- + +## 4. Files Changed Summary + +| Action | File | +| ------------- | --------------------------------------------------------------------------------------------------- | +| **Modify** | `EmbeddedSQLTester/EmbeddedSQLTester.csproj` — swap NuGet package | +| **Create** | `SQLitePlatformConversion/StatementConverters/SqlToken.cs` — token wrapper + extraction helper | +| **Modify** | `SQLitePlatformConversion/SQLServerToOrmliteSQLiteDialectConverter.cs` — new parser, AST navigation | +| **Modify** | `StatementConverters/ClauseProcessors/ClauseConverterBase.cs` — accept `List` | +| **Modify** | `StatementConverters/ExpressionProcessors/ExpressionConverterBase.cs` — accept `List` | +| **Modify** | `StatementConverters/ClauseProcessors/GeneralClauseConverter.cs` — constructor signature | +| **Modify** | `StatementConverters/ClauseProcessors/FromClauseConverter.cs` — constructor signature | +| **Modify** | `StatementConverters/ClauseProcessors/WhereClauseConverter.cs` — constructor signature | +| **Modify** | `StatementConverters/ClauseProcessors/OrderByClauseConverter.cs` — constructor signature | +| **Modify** | `StatementConverters/ClauseProcessors/InsertClauseConverter.cs` — constructor signature | +| **Modify** | `StatementConverters/ClauseProcessors/UpdateClauseConverter.cs` — constructor signature | +| **Modify** | `StatementConverters/LimitStatementConverter.cs` — constructor signature | +| **Modify** | `StatementConverters/ExpressionProcessors/ValuesExpressionConverter.cs` — constructor signature | +| **Modify** | `StatementConverters/SelectStatementConverter.cs` — AST navigation + token extraction | +| **Modify** | `StatementConverters/UpdateStatementConverter.cs` — AST navigation + token extraction | +| **Modify** | `StatementConverters/InsertStatementConverter.cs` — AST navigation + token extraction | +| **Modify** | `StatementConverters/DeleteStatementConverter.cs` — AST navigation + token extraction | +| **No change** | All test files (`EmbeddedSQLTester.Tests/`) | +| **No change** | `SQLitePlatformConversion/MyExtensions.cs` | + +--- + +## 5. Risk & Mitigation + +| Risk | Mitigation | +| ------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| ScriptDom token offsets differ from TSQL.Parser, breaking whitespace reconstruction | Simplify spacing logic: insert a single space between every non-empty token output, or normalize offsets in the `SqlToken` wrapper | +| ScriptDom groups tokens differently (e.g., `@Parameter` may be one or two tokens) | Unit tests will catch mismatches immediately; adjust token extraction boundaries | +| ScriptDom may not parse malformed SQL the same way (e.g., unmatched `)` in LIKE test) | If ScriptDom rejects, the specific SQL might need minor input normalization or the test expected output may need review (but tests are not to be changed, so the converter must handle it) | +| `TOP` token may not appear in the same token range | Explicitly include `TopRowFilter` tokens when building the SELECT clause token list | + +--- + +## 6. Execution Order + +1. Step 1 (NuGet) → Step 2 (SqlToken wrapper) → Step 3-4 (base classes) → Step 5 (leaf converters) → Step 6-8 (statement converters + main converter + extraction) → Step 9 (edge cases) → Step 10 (test) + +Estimated scope: ~17 files touched, 1 new file created, 0 test files modified.