diff --git a/ColumnExtractor/Parser.cs b/ColumnExtractor/Parser.cs index ee9d3e9..4239059 100644 --- a/ColumnExtractor/Parser.cs +++ b/ColumnExtractor/Parser.cs @@ -12,20 +12,22 @@ public class Parser private readonly bool _log; private readonly bool _outputSelectStar; private readonly bool _bracketOutput; + private readonly bool _outputPivotGeneratedColumns; - public Parser() + public Parser() { _log = false; _outputSelectStar = false; _bracketOutput = false; } - public Parser(bool log, bool outputSelectStar, bool bracketOutput) + public Parser(bool log, bool outputSelectStar, bool bracketOutput, bool outputPivotGeneratedColumns = false) { _log = log; _outputSelectStar = outputSelectStar; _bracketOutput = bracketOutput; - } + _outputPivotGeneratedColumns = outputPivotGeneratedColumns; + } public ParsedData GetColumns(string sql) { @@ -37,7 +39,9 @@ public ParsedData GetColumns(string sql) var result = parser.Parse(reader, out var errors); var traverser = new Traverser(_log); var columnData = traverser.TraverseObject(result, null).Distinct().ToArray(); - var tableData = columnData.Where(c => c?.AmbiguousTableReferences != null) + var tableData = columnData + .Where(c => _outputPivotGeneratedColumns || !c.IsPivotGeneratedColumn) + .Where(c => c?.AmbiguousTableReferences != null) .SelectMany(c => c.AmbiguousTableReferences) .Concat(columnData.Select(c => c.AbsoluteTableReference)) .Where(c => c != null) @@ -87,6 +91,7 @@ public TableData[] GetTables(string sql) foreach (var table in tableData) { table.Columns = columnData + .Where(c => _outputPivotGeneratedColumns || !c.IsPivotGeneratedColumn) .Where(c => c.AbsoluteTableReference != null && table.Equals(new TableData(c.AbsoluteTableReference, _bracketOutput))) .Select(c => new ColumnData(c, _bracketOutput)) .Where(cd => _outputSelectStar || cd.ColumnNM != "*") @@ -94,7 +99,8 @@ public TableData[] GetTables(string sql) .ToArray(); table.PossibleColumns = columnData - .Where(c => c.AmbiguousTableReferences != null && c.AmbiguousTableReferences.Any(t => table.Equals(new TableData(t, _bracketOutput)))) + .Where(c => _outputPivotGeneratedColumns || !c.IsPivotGeneratedColumn) + .Where(c => c.AmbiguousTableReferences != null && c.AmbiguousTableReferences.Any(t => table.Equals(new TableData(t, _bracketOutput)))) .Select(c => new ColumnData(c, _bracketOutput)) .Where(cd => _outputSelectStar || cd.ColumnNM != "*") .Distinct() diff --git a/ColumnExtractor/Traverse/Column.cs b/ColumnExtractor/Traverse/Column.cs index 103aced..52c79ba 100644 --- a/ColumnExtractor/Traverse/Column.cs +++ b/ColumnExtractor/Traverse/Column.cs @@ -11,6 +11,8 @@ public class Column public Table AbsoluteTableReference; public IEnumerable AmbiguousTableReferences; public IEnumerable CteReferences; + public bool IsPivotGeneratedColumn; + public string FullyQualifiedName => GetFullyQualifiedName(true); public string GetFullyQualifiedName(bool brackets) @@ -27,22 +29,29 @@ public string GetFullyQualifiedName(bool brackets) : availableParts.Any() ? $"{string.Join(".", availableParts)}" : null; } - public override bool Equals(object obj) - { - return obj is Column summary && - Name.Equals(summary.Name, StringComparison.OrdinalIgnoreCase) && - EqualityComparer
.Default.Equals(AbsoluteTableReference, summary.AbsoluteTableReference) && - EqualityComparer>.Default.Equals(AmbiguousTableReferences, - summary.AmbiguousTableReferences); - } + protected bool Equals(Column other) + { + return string.Equals(Name, other.Name) && Equals(AbsoluteTableReference, other.AbsoluteTableReference) && Equals(AmbiguousTableReferences, other.AmbiguousTableReferences) && IsPivotGeneratedColumn == other.IsPivotGeneratedColumn; + } - public override int GetHashCode() - { - var hashCode = 1102383335; - hashCode = hashCode * -1521134295 + EqualityComparer.Default.GetHashCode(Name != null ? Name.ToLowerInvariant() : null); - hashCode = hashCode * -1521134295 + EqualityComparer
.Default.GetHashCode(AbsoluteTableReference); - hashCode = hashCode * -1521134295 + EqualityComparer>.Default.GetHashCode(AmbiguousTableReferences); - return hashCode; - } - } + public override bool Equals(object obj) + { + if (ReferenceEquals(null, obj)) return false; + if (ReferenceEquals(this, obj)) return true; + if (obj.GetType() != this.GetType()) return false; + return Equals((Column) obj); + } + + public override int GetHashCode() + { + unchecked + { + var hashCode = (Name != null ? Name.GetHashCode() : 0); + hashCode = (hashCode * 397) ^ (AbsoluteTableReference != null ? AbsoluteTableReference.GetHashCode() : 0); + hashCode = (hashCode * 397) ^ (AmbiguousTableReferences != null ? AmbiguousTableReferences.GetHashCode() : 0); + hashCode = (hashCode * 397) ^ IsPivotGeneratedColumn.GetHashCode(); + return hashCode; + } + } + } } \ No newline at end of file diff --git a/ColumnExtractor/Traverse/Cte.cs b/ColumnExtractor/Traverse/Cte.cs index 8841f3b..3384f24 100644 --- a/ColumnExtractor/Traverse/Cte.cs +++ b/ColumnExtractor/Traverse/Cte.cs @@ -4,21 +4,18 @@ namespace ColumnExtractor.Traverse { public class Cte { - public string Alias; public string Name; public List
LinkedTables = new List
(); public override bool Equals(object obj) { return obj is Cte summary && - Alias == summary.Alias && Name == summary.Name; } public override int GetHashCode() { var hashCode = -601341547; - hashCode = hashCode * -1521134295 + EqualityComparer.Default.GetHashCode(Alias); hashCode = hashCode * -1521134295 + EqualityComparer.Default.GetHashCode(Name != null ? Name.ToLowerInvariant() : null); return hashCode; } diff --git a/ColumnExtractor/Traverse/Traverser.cs b/ColumnExtractor/Traverse/Traverser.cs index 4d92b6e..e2053ed 100644 --- a/ColumnExtractor/Traverse/Traverser.cs +++ b/ColumnExtractor/Traverse/Traverser.cs @@ -149,8 +149,12 @@ public IEnumerable TraverseObject(object obj, Cte[] ctes, Table[] parent } } - return columns; - } + // Remove pivot generated column duplicates + return columns + .OrderByDescending(c => c.IsPivotGeneratedColumn) + .GroupBy(c => c.FullyQualifiedName) + .Select(grp => grp.First()); + } public Table GetTableFromReference(NamedTableReference namedTableReference) { @@ -169,7 +173,7 @@ public Table GetTableFromReference(NamedTableReference namedTableReference) return table; } - public Column GetColumnFromIdentifiers(string[] identifiers, string alias, IEnumerable
containers, IEnumerable
parentContainers, IEnumerable ctes, bool inPivot = false) + public Column GetColumnFromIdentifiers(string[] identifiers, string alias, IEnumerable
containers, IEnumerable
parentContainers, IEnumerable ctes, bool inPivot = false) { if (identifiers == null) return null; @@ -177,7 +181,7 @@ public Column GetColumnFromIdentifiers(string[] identifiers, string alias, IEnum if (parentContainers == null) parentContainers = new Table[0]; if (ctes == null) ctes = new Cte[0]; - Dump($"{_pad}PARSING COLUMN: tables: { string.Join(", ", containers.Count()) }, parent tables: { string.Join(", ", parentContainers.Count()) }, ctes: { string.Join(", ", ctes.Count()) }"); + Dump($"{_pad}PARSING COLUMN: tables: { string.Join(", ", containers.Select(c => c.Name)) }, parent tables: { string.Join(", ", parentContainers.Select(c => c.Name)) }, ctes: { string.Join(", ", ctes.Select(c => c.Name)) }"); var allTables = containers.Concat(parentContainers).Distinct().ToArray(); @@ -194,20 +198,24 @@ public Column GetColumnFromIdentifiers(string[] identifiers, string alias, IEnum if (identifiers.Length == 1) { - matches.AddRange(containers); cteMatches.AddRange(ctes); linkedMatches.AddRange(cteMatches.Where(cte => cte.LinkedTables.Count == 1 && (cte.LinkedTables.First().SelectColumns.Contains(column.Name.ToLowerInvariant()) - || cte.LinkedTables.First().SelectColumns.Contains("*") - || inPivot)) - .SelectMany(cte => cte.LinkedTables) - .Distinct()); - matches.AddRange(cteMatches.Where(cte => cte.LinkedTables.Any(lt => - lt.PossibleSelectColumns.Contains(column.Name.ToLowerInvariant()) - || lt.PossibleSelectColumns.Contains("*") + || inPivot)) .SelectMany(cte => cte.LinkedTables) .Distinct()); + + if (!linkedMatches.Any()) + { + matches.AddRange(containers); + matches.AddRange(cteMatches.Where(cte => cte.LinkedTables.Any(lt => + lt.PossibleSelectColumns.Contains(column.Name.ToLowerInvariant()) + || lt.PossibleSelectColumns.Contains("*") + || inPivot)) + .SelectMany(cte => cte.LinkedTables) + .Distinct()); + } } else if (identifiers.Length == 2) { @@ -226,24 +234,26 @@ public Column GetColumnFromIdentifiers(string[] identifiers, string alias, IEnum matches.Add(aliasMatches.First()); } - cteMatches.AddRange(ctes.Where(cte => !string.IsNullOrWhiteSpace(cte.Alias) && cte.Alias.Equals(tableNameOrAlias, StringComparison.OrdinalIgnoreCase) - || cte.Alias == null && cte.Name.Equals(tableNameOrAlias, StringComparison.OrdinalIgnoreCase))); + cteMatches.AddRange(ctes); + //cteMatches.AddRange(ctes.Where(cte => !string.IsNullOrWhiteSpace(cte.Name) && cte.Name.Equals(tableNameOrAlias, StringComparison.OrdinalIgnoreCase))); linkedMatches.AddRange(cteMatches .Where(cte => cte.LinkedTables.Count == 1 && (cte.LinkedTables.First().SelectColumns.Contains(column.Name.ToLowerInvariant()) - || cte.LinkedTables.First().SelectColumns.Contains("*") || inPivot)) .SelectMany(cte => cte.LinkedTables) .Distinct()); - matches.AddRange(cteMatches.Where(cte => cte.LinkedTables.Any(lt => - lt.PossibleSelectColumns.Contains(column.Name.ToLowerInvariant()) - || lt.PossibleSelectColumns.Contains("*") - || inPivot)) - .SelectMany(cte => cte.LinkedTables) - .Distinct()); - } + if (!linkedMatches.Any()) + { + matches.AddRange(cteMatches.Where(cte => cte.LinkedTables.Any(lt => + lt.PossibleSelectColumns.Contains(column.Name.ToLowerInvariant()) + || lt.PossibleSelectColumns.Contains("*") + || inPivot)) + .SelectMany(cte => cte.LinkedTables) + .Distinct()); + } + } else if (identifiers.Length == 3) { var schemaName = identifiers[0]; @@ -279,9 +289,13 @@ public Column GetColumnFromIdentifiers(string[] identifiers, string alias, IEnum .Select(m => m.Name.ToLowerInvariant()).ToArray(); cteMatches.AddRange(ctes.Where(cte => tableNames.Contains(cte.Name.ToLowerInvariant()))); - if (cteMatches.Any()) + cteMatches = cteMatches.Distinct().ToList(); + linkedMatches = linkedMatches.Distinct().ToList(); + matches = matches.Distinct().ToList(); + + if (cteMatches.Any()) { - column.CteReferences = cteMatches.Select(t => new Cte { Name = t.Name, Alias = t.Alias }).ToArray(); + column.CteReferences = cteMatches.Select(t => new Cte { Name = t.Name }).ToArray(); var cteNames = ctes.Select(cte => cte.Name.ToLowerInvariant()).ToArray(); var reject = matches.Where(m => m.Server == null && m.Database == null && m.Schema == null && cteNames.Contains(m.Name.ToLowerInvariant())); @@ -299,6 +313,14 @@ public Column GetColumnFromIdentifiers(string[] identifiers, string alias, IEnum Dump($"{_pad}> FOUND TABLES (MATCHES: {matches.Count}): [{ string.Join(", ", matches.Select(c => c.FullyQualifiedName)) }]"); if (matches.Count > 1) column.AmbiguousTableReferences = matches; else if (matches.Count == 1) column.AbsoluteTableReference = matches.First(); + else if (cteMatches.Count == 1 && cteMatches.First().LinkedTables.Count == 1) + { + column.AbsoluteTableReference = cteMatches.First().LinkedTables.First(); + } + else if (cteMatches.SelectMany(c => c.LinkedTables).Distinct().Count() == 1) + { + column.AbsoluteTableReference = cteMatches.SelectMany(c => c.LinkedTables).Distinct().First(); + } } var matchingTable = column.AbsoluteTableReference != null @@ -374,15 +396,18 @@ public Tuple, IEnumerable, IEnumerable
> HandleCt } cte.LinkedTables = tables.Distinct().ToList(); + foreach (var linkedTable in cte.LinkedTables) { linkedTable.SelectColumns = columns .Where(c => ReferenceEquals(c.AbsoluteTableReference, linkedTable)).Select(c => c.Name.ToLowerInvariant()) + .Distinct() .ToArray(); linkedTable.PossibleSelectColumns = columns .Where(c => c.AmbiguousTableReferences != null && c.AmbiguousTableReferences.Any(t => ReferenceEquals(t, linkedTable))) .Select(c => c.Name.ToLowerInvariant()) - .ToArray(); + .Distinct() + .ToArray(); } if (cte.LinkedTables.Any()) Dump($"{_pad}FOUND CTE-LINKED TABLES: {cte.Name}, tables: {string.Join(", ", cte.LinkedTables.Select(t => t.FullyQualifiedName))}"); @@ -426,8 +451,8 @@ public Tuple, IEnumerable
> HandleQuery(object obj, Ct tables.AddRange(result.Item2); } } - - var fromClause = Helpers.GetPropertyValueWithName(obj, "FromClause"); + + var fromClause = Helpers.GetPropertyValueWithName(obj, "FromClause"); if (fromClause != null) { var tableReferences = Helpers.GetEnumerablePropertyValueWithName(fromClause, "TableReferences"); @@ -450,7 +475,14 @@ public Tuple, IEnumerable
> HandleQuery(object obj, Ct Dump($"{_pad}PROPERTY [SelectElements]: handling columns..."); foreach (var element in selectElements) { - columns.AddRange(HandleColumns(element, null, tables.ToArray(), parentTables, ctes)); + if (element is SelectScalarExpression selectScalarExpression && selectScalarExpression.Expression is StringLiteral) + { + columns.Add(new Column { Name = selectScalarExpression.ColumnName.Value }); + } + else + { + columns.AddRange(HandleColumns(element, null, tables.ToArray(), parentTables, ctes)); + } } } @@ -557,7 +589,7 @@ public Tuple, IEnumerable
> HandleTables(object obj, Cte[ { tables.Add(GetTableFromReference(obj as NamedTableReference)); } - else if (obj.GetType().IsAssignableFrom(typeof(QueryDerivedTable))) + else if (obj.GetType().IsAssignableFrom(typeof(QueryDerivedTable))) { var derived = obj as QueryDerivedTable; Dump($"{_pad}ANALYZING QueryDerivedTable..."); @@ -566,24 +598,48 @@ public Tuple, IEnumerable
> HandleTables(object obj, Cte[ { Dump($"{_pad}PROPERTY [QueryExpression]: handling..."); var result = HandleQuery(derived.QueryExpression, parentCtes, parentTables); - var cte = new Cte { Name = derived.Alias?.Value, LinkedTables = result.Item2.Distinct().ToList() }; + var cte = new Cte { Name = derived.Alias?.Value, LinkedTables = result.Item2.Distinct().ToList() }; foreach (var linkedTable in cte.LinkedTables) { linkedTable.SelectColumns = result.Item1 .Where(c => ReferenceEquals(c.AbsoluteTableReference, linkedTable)).Select(c => c.Name.ToLowerInvariant()) - .ToArray(); + .Distinct() + .ToArray(); linkedTable.PossibleSelectColumns = result.Item1 .Where(c => c.AmbiguousTableReferences != null && c.AmbiguousTableReferences.Any(t => ReferenceEquals(t, linkedTable))) .Select(c => c.Name.ToLowerInvariant()) - .ToArray(); + .Distinct() + .ToArray(); } if (cte.LinkedTables.Any()) Dump($"{_pad}FOUND DERIVED-LINKED TABLES: {cte.Name}, tables: {string.Join(", ", cte.LinkedTables.Select(t => t.FullyQualifiedName))}"); ctes.Add(cte); } } - else + else if (obj.GetType().IsAssignableFrom(typeof(UnpivotedTableReference))) + { + if (obj is UnpivotedTableReference unpivoted) + { + var linkedTables = HandleTables(unpivoted.TableReference, parentCtes, parentTables); + var cteLinkedTables = linkedTables.Item1.SelectMany(c => c.LinkedTables).Concat(linkedTables.Item2); + + ctes.AddRange(linkedTables.Item1); + ctes.Add(new Cte { Name = unpivoted.Alias.Value, LinkedTables = cteLinkedTables.ToList() }); + } + } + else if (obj.GetType().IsAssignableFrom(typeof(PivotedTableReference))) + { + if (obj is PivotedTableReference pivoted) + { + var linkedTables = HandleTables(pivoted.TableReference, parentCtes, parentTables); + var cteLinkedTables = linkedTables.Item1.SelectMany(c => c.LinkedTables).Concat(linkedTables.Item2); + + ctes.AddRange(linkedTables.Item1); + ctes.Add(new Cte { Name = pivoted.Alias.Value, LinkedTables = cteLinkedTables.ToList() }); + } + } + else { var references = Helpers.GetPropertiesWithNames(obj, "FirstTableReference", "SecondTableReference", "TableReference", "Join").ToArray(); if (references.Any()) _level++; @@ -627,10 +683,6 @@ public IEnumerable HandleColumns(object obj, string alias, Table[] table handleColumns.AddRange(HandleProperties(obj, alias, tables, parentTables, ctes)); } } - else if (obj.GetType().IsAssignableFrom(typeof(ColumnReferenceExpression))) - { - handleColumns.Add(GetColumnFromReference(obj as ColumnReferenceExpression, alias, tables, parentTables, ctes)); - } else if (obj.GetType().IsAssignableFrom(typeof(SelectStarExpression))) { handleColumns.Add(GetColumnFromStarReference(obj as SelectStarExpression, tables, parentTables, ctes)); @@ -639,13 +691,35 @@ public IEnumerable HandleColumns(object obj, string alias, Table[] table { if (obj is PivotedTableReference pivotedTableReference) { - handleColumns.Add(GetColumnFromReference(pivotedTableReference.PivotColumn, null, tables, parentTables, ctes, true)); - if (pivotedTableReference.ValueColumns != null) + string tableName = null; + string tableAlias = null; + + if (pivotedTableReference.TableReference is NamedTableReference namedTableReference) + { + tableName = namedTableReference.SchemaObject.BaseIdentifier.Value; + tableAlias = namedTableReference.Alias?.Value; + } + if (pivotedTableReference.TableReference is QueryDerivedTable queryDerivedTable) + { + tableAlias = queryDerivedTable.Alias?.Value; + } + + handleColumns.Add(GetColumnFromReference(pivotedTableReference.PivotColumn, null, tables, parentTables, ctes, true)); + + if (pivotedTableReference.PivotColumn != null) + { + var identifiers = new[] { tableAlias ?? tableName }.Concat(pivotedTableReference.PivotColumn.MultiPartIdentifier.Identifiers.Select(i => i.Value)).ToArray(); + var column = GetColumnFromIdentifiers(identifiers, null, tables, parentTables, ctes, true); + handleColumns.Add(column); + } + + if (pivotedTableReference.ValueColumns != null) { foreach (var valueColumn in pivotedTableReference.ValueColumns) - { - handleColumns.Add(GetColumnFromReference(valueColumn, null, tables, parentTables, ctes, true)); - } + { + var column = GetColumnFromReference(valueColumn, null, tables, parentTables, ctes, true); + handleColumns.Add(column); + } } } @@ -654,20 +728,54 @@ public IEnumerable HandleColumns(object obj, string alias, Table[] table else if (obj.GetType().IsAssignableFrom(typeof(UnpivotedTableReference))) { if (obj is UnpivotedTableReference unpivotedTableReference) - { - if (unpivotedTableReference.PivotColumn?.Value != null) - { - handleColumns.Add(GetColumnFromIdentifiers(new[] { unpivotedTableReference.PivotColumn.Value }, null, tables, parentTables, ctes, true)); - } - if (unpivotedTableReference.ValueColumn?.Value != null) - { - handleColumns.Add(GetColumnFromIdentifiers(new[] { unpivotedTableReference.ValueColumn.Value }, null, tables, parentTables, ctes, true)); - } + { + string tableName = null; + string tableAlias = null; + + if (unpivotedTableReference.TableReference is NamedTableReference namedTableReference) + { + tableName = namedTableReference.SchemaObject.BaseIdentifier.Value; + tableAlias = namedTableReference.Alias?.Value; + } + if (unpivotedTableReference.TableReference is QueryDerivedTable queryDerivedTable) + { + tableAlias = queryDerivedTable.Alias?.Value; + } + + if (unpivotedTableReference.PivotColumn?.Value != null) + { + var column = GetColumnFromIdentifiers(new[] { tableAlias ?? tableName, unpivotedTableReference.PivotColumn.Value }, null, tables, parentTables, ctes, true); + column.IsPivotGeneratedColumn = true; + Dump($"{_pad}* UNPIVOT GENERATED COLUMN"); + column.AbsoluteTableReference.SelectColumns = column.AbsoluteTableReference.SelectColumns.Concat(new[] { column.Name.ToLowerInvariant() }).Distinct().ToArray(); + handleColumns.Add(column); + } + if (unpivotedTableReference.ValueColumn?.Value != null) + { + var column = GetColumnFromIdentifiers(new[] { tableAlias ?? tableName, unpivotedTableReference.ValueColumn.Value }, null, tables, parentTables, ctes, true); + column.IsPivotGeneratedColumn = true; + Dump($"{_pad}* UNPIVOT GENERATED COLUMN"); + column.AbsoluteTableReference.SelectColumns = column.AbsoluteTableReference.SelectColumns.Concat(new[] { column.Name.ToLowerInvariant() }).Distinct().ToArray(); + handleColumns.Add(column); + } + if (unpivotedTableReference.InColumns?.Count > 0) + { + foreach (var inColumn in unpivotedTableReference.InColumns) + { + var identifiers = new [] { tableAlias ?? tableName }.Concat(inColumn.MultiPartIdentifier.Identifiers.Select(i => i.Value)).ToArray(); + + handleColumns.Add(GetColumnFromIdentifiers(identifiers, null, tables, parentTables, ctes, true)); + } + } } handleColumns.AddRange(HandleProperties(obj, alias, tables, parentTables, ctes)); } - else if (obj is QueryExpression) + else if (obj.GetType().IsAssignableFrom(typeof(ColumnReferenceExpression))) + { + handleColumns.Add(GetColumnFromReference(obj as ColumnReferenceExpression, alias, tables, parentTables, ctes)); + } + else if (obj is QueryExpression) { Dump($"{_pad}HANDLING QueryExpression..."); handleColumns.AddRange(HandleQuery(obj, ctes, tables).Item1); @@ -695,7 +803,7 @@ public IEnumerable HandleProperties(object obj, string alias, Table[] ta if (subQuery != null) { Dump($"{_pad}Is SubQuery..."); - var columns = TraverseObject(subQuery, ctes, tables.ToArray()); + var columns = TraverseObject(subQuery, ctes, tables); handleColumns.AddRange(columns); } @@ -706,12 +814,12 @@ public IEnumerable HandleProperties(object obj, string alias, Table[] ta var properties = new[] { - "FirstQueryExpression", "SecondQueryExpression", "FirstTableReference", "SecondTableReference", "TableReference", "Join", - "FirstExpression", "SecondExpression", "WhenExpression", "ThenExpression", "ElseExpression", "Expression", "Expressions", - "FirstExpression", "SecondExpression", "ThirdExpression", "QueryExpression", "Parameter", "SearchCondition", "Parameters", - "WhenClauses", "OrderByClause", "OrderByElements", "OverClause", "InputExpression", "WithinGroupClause", "Partitions", - "WindowFrameClause", "WindowDelimiter", "OffsetValue", "Top", "Bottom", "Predicate", "Values" - }; + "FirstQueryExpression", "SecondQueryExpression", "FirstTableReference", "SecondTableReference", "TableReference", "Join", + "FirstExpression", "SecondExpression", "WhenExpression", "ThenExpression", "ElseExpression", "Expression", "Expressions", + "FirstExpression", "SecondExpression", "ThirdExpression", "QueryExpression", "Parameter", "SearchCondition", "Parameters", + "WhenClauses", "OrderByClause", "OrderByElements", "OverClause", "InputExpression", "WithinGroupClause", "Partitions", + "WindowFrameClause", "WindowDelimiter", "OffsetValue", "Top", "Bottom", "Predicate", "Values" + }; var references = Helpers.GetPropertiesWithNames(obj, properties.Except(skipProperties).ToArray()).ToArray(); diff --git a/SqlParser.linq b/SqlParser.linq index e9554a9..bc61d4b 100644 --- a/SqlParser.linq +++ b/SqlParser.linq @@ -1,25 +1,49 @@ - C:\SC\SQL Parser\ColumnExtractor\bin\Release\ColumnExtractor.dll + C:\Source\Catalyst.SqlUtilities\ColumnExtractor\bin\Debug\ColumnExtractor.dll + C:\Source\Catalyst.SqlUtilities\ColumnExtractor\bin\Debug\Microsoft.SqlServer.TransactSql.ScriptDom.dll ColumnExtractor Microsoft.SqlServer.TransactSql.ScriptDom + ColumnExtractor.Traverse + ColumnExtractor.Models void Main() { - var sql = @"SELECT - DATEADD(D,Something,CAST('2011-09-19 16:52:00.000' AS DATE)) - FROM SAM.InfectiousDisease.CAUTI26MetricCensusPatientDay a - INNER JOIN Foo f ON f.id = a.id + var sql = @" +SELECT + year(invoiceDate) as [year],left(datename(month,invoicedate),3)as [month], + InvoiceAmount as Amount + FROM Invoice +PIVOT +( + SUM(Amount) + FOR [month] IN (jan, feb, mar, apr, may, jun, jul, aug, sep, oct, nov, dec) +)AS pvt "; - //sql.Dump("SQL"); - var parser = new Parser(true, true, true); + Dump(sql); + var parser = new Parser(true, true, true, false); + var traverser = new Traverser(true); var results = parser.GetTables(sql); parser.GetErrors(sql).Dump("ERRORS"); - //parser.GetStatements(sql).Dump("STATEMENTS"); + parser.GetStatements(sql).Dump("STATEMENTS"); results.Dump("RESULTS"); results.SelectMany(r => r.Columns).Select(c => c.FullyQualifiedNM).Dump("COLUMNS"); results.SelectMany(r => r.PossibleColumns).Select(c => c.FullyQualifiedNM).Distinct().Dump("UNKNOWN COLUMNS"); +} + +private void Dump(string sql) +{ + var parser = new TSql140Parser(true); + var parsedData = new ParsedData(); + + using (var reader = new StringReader(sql)) + { + var result = parser.Parse(reader, out var errors); + var traverser = new Traverser(false); + var columnData = traverser.TraverseObject(result, null).Distinct().ToArray(); + columnData.Dump(); + } } \ No newline at end of file diff --git a/UnitTests/Tests.cs b/UnitTests/Tests.cs index 19172f8..b480fae 100644 --- a/UnitTests/Tests.cs +++ b/UnitTests/Tests.cs @@ -22,179 +22,16 @@ public void SmokeTest() public void Query1() { var sql = @" -WITH -LinkMeasureElementCTE AS ( - SELECT * - FROM SAM.Pneumonia.QVExtElementBASE -), + WITH + LinkMeasureElementCTE AS (SELECT * FROM SAM.Pneumonia.QVExtElementBASE), + ConfigReplaceCTE AS (SELECT [ExpressionCD] = 'Base'), + StandardExpressionCTE AS (SELECT ElementCD FROM LinkMeasureElementCTE CROSS JOIN ConfigReplaceCTE c) -ConfigReplaceCTE AS ( - SELECT - [ExpressionCD] = 'Base', - [ReplaceSetAnalysisWithSTR] = '', - [ReplaceAggrWithSTR] = '@(=%DateCycleGroupByCOL)' - UNION ALL - SELECT - [ExpressionCD] = 'Total', - [ReplaceSetAnalysisWithSTR] = 'TOTAL ', - [ReplaceAggrWithSTR] = '@(=%DateCycleGroupByCOL)' - UNION ALL - SELECT - [ExpressionCD] = 'PeriodCurrent', - [ReplaceSetAnalysisWithSTR] = '{<@(=@(xPeriodCurrent))>}', - [ReplaceAggrWithSTR] = '@(=%MeasureRangeGroupByCOL)' - UNION ALL - SELECT - [ExpressionCD] = 'PeriodPrior', - [ReplaceSetAnalysisWithSTR] = '{<@(=@(xPeriodPrior))>}', - [ReplaceAggrWithSTR] = '@(=%MeasureRangeGroupByCOL)' -), - - - -/** add standard set analysis expressions **/ -StandardExpressionCTE AS ( - SELECT - MeasureID - ,ElementCD - ,c.ExpressionCD AS ExpressionCD - ,REPLACE(REPLACE(ExpressionSTR,SetAnalysisSTR,c.ReplaceSetAnalysisWithSTR),AggrSTR,c.ReplaceAggrWithSTR) AS ExpressionSTR - FROM LinkMeasureElementCTE - CROSS JOIN ConfigReplaceCTE c - - UNION ALL - SELECT - MeasureID - ,ElementCD - ,'BaseChng' AS ExpressionCD - ,'NUM(@(=ONLY({<%MeasureID={'+CAST(MeasureID AS VARCHAR)+'},%ElementCD={'+ElementCD+'},%ExpressionCD={Base}>}%ExpressionSTR))-@(=ONLY({<%MeasureID={'+CAST(MeasureID AS VARCHAR)+'},%ElementCD={'+ElementCD+'},%ExpressionCD={Total}>}%ExpressionSTR)),chr(9650)&'+CHAR(39)+MeasureFormatCD+';'+CHAR(39)+'&chr(9660)&'+CHAR(39)+MeasureFormatCD+CHAR(39)+')' AS ExpressionSTR - FROM LinkMeasureElementCTE - UNION ALL - SELECT - MeasureID - ,ElementCD - ,'BasePcntChng' AS ExpressionCD - ,'NUM((@(=ONLY({<%MeasureID={'+CAST(MeasureID AS VARCHAR)+'},%ElementCD={'+ElementCD+'},%ExpressionCD={Base}>}%ExpressionSTR))-@(=ONLY({<%MeasureID={'+CAST(MeasureID AS VARCHAR)+'},%ElementCD={'+ElementCD+'},%ExpressionCD={Total}>}%ExpressionSTR)))/(@(=ONLY({<%MeasureID={'+CAST(MeasureID AS VARCHAR)+'},%ElementCD={'+ElementCD+'},%ExpressionCD={Base}>}%ExpressionSTR))),'+CHAR(39)+'#,##0%'+CHAR(39)+')' AS ExpressionSTR - FROM LinkMeasureElementCTE - - UNION ALL - SELECT - MeasureID - ,ElementCD - ,'PeriodChng' AS ExpressionCD - ,'NUM(@(=ONLY({<%MeasureID={'+CAST(MeasureID AS VARCHAR)+'},%ElementCD={'+ElementCD+'},%ExpressionCD={PeriodCurrent}>}%ExpressionSTR))-@(=ONLY({<%MeasureID={'+CAST(MeasureID AS VARCHAR)+'},%ElementCD={'+ElementCD+'},%ExpressionCD={PeriodPrior}>}%ExpressionSTR)),chr(9650)&'+CHAR(39)+MeasureFormatCD+';'+CHAR(39)+'&chr(9660)&'+CHAR(39)+MeasureFormatCD+CHAR(39)+')' AS ExpressionSTR - FROM LinkMeasureElementCTE - UNION ALL - SELECT - MeasureID - ,ElementCD - ,'PeriodPcntChng' AS ExpressionCD - ,'NUM((@(=ONLY({<%MeasureID={'+CAST(MeasureID AS VARCHAR)+'},%ElementCD={'+ElementCD+'},%ExpressionCD={PeriodCurrent}>}%ExpressionSTR))-@(=ONLY({<%MeasureID={'+CAST(MeasureID AS VARCHAR)+'},%ElementCD={'+ElementCD+'},%ExpressionCD={PeriodPrior}>}%ExpressionSTR)))/(@(=ONLY({<%MeasureID={'+CAST(MeasureID AS VARCHAR)+'},%ElementCD={'+ElementCD+'},%ExpressionCD={PeriodCurrent}>}%ExpressionSTR))),'+CHAR(39)+'#,##0%'+CHAR(39)+')' AS ExpressionSTR - FROM LinkMeasureElementCTE -), - -/** add additional elements for informational purposes **/ -InfoExpressionCTE AS ( - SELECT - MeasureID - ,'LABEL' AS ElementCD - ,'Base' AS ExpressionCD - ,CHAR(39)+MeasureNM+CHAR(39) AS ExpressionSTR - FROM LinkMeasureElementCTE - UNION - SELECT - MeasureID - ,'UNIT' AS ElementCD - ,'Base' AS ExpressionCD - ,CHAR(39)+MeasureUnitDSC+CHAR(39) AS ExpressionSTR - FROM LinkMeasureElementCTE - UNION - SELECT - MeasureID - ,'UNIT_NUMER' AS ElementCD - ,'Base' AS ExpressionCD - ,CHAR(39)+NumUnitDSC+CHAR(39) AS ExpressionSTR - FROM LinkMeasureElementCTE - UNION - SELECT - MeasureID - ,'UNIT_DENOM' AS ElementCD - ,'Base' AS ExpressionCD - ,CHAR(39)+DenomUnitDSC+CHAR(39) AS ExpressionSTR - FROM LinkMeasureElementCTE - UNION - SELECT - MeasureID - ,'ICON' AS ElementCD - ,'Base' AS ExpressionCD - ,CHAR(39)+'qmem://ImageID/'+ISNULL(MeasureIconNM,'')+CHAR(39) AS ExpressionSTR - FROM LinkMeasureElementCTE - UNION - SELECT - MeasureID - ,'ICON_GHOST' AS ElementCD - ,'Base' AS ExpressionCD - ,CHAR(39)+'qmem://ImageID/'+ISNULL(MeasureIconNM+'_ghost','')+CHAR(39) AS ExpressionSTR - FROM LinkMeasureElementCTE - UNION - SELECT - MeasureID - ,'ICON_TYPE' AS ElementCD - ,'Base' AS ExpressionCD - ,CASE MeasureTypeDSC - WHEN 'Outcome' THEN CHAR(39)+'qmem://ImageID/outcome_rounded'+CHAR(39) - WHEN 'Process' THEN CHAR(39)+'qmem://ImageID/process_rounded'+CHAR(39) - END AS ExpressionSTR - FROM LinkMeasureElementCTE - UNION - SELECT - MeasureID - ,'TYPE' AS ElementCD - ,'Base' AS ExpressionCD - ,CHAR(39)+MeasureTypeDSC+CHAR(39) AS ExpressionSTR - FROM LinkMeasureElementCTE - - UNION - SELECT - MeasureID - ,ElementCD - ,'BaseChng_Color' AS ExpressionCD - ,CASE - WHEN MeasureDirectionCD = 'n/a' - THEN 'none' - WHEN MeasureDirectionCD = 'Up' - THEN 'IF( @(=ONLY({<%MeasureID={'+CAST(MeasureID AS VARCHAR)+'},%ElementCD={'+ElementCD+'},%ExpressionCD={BaseChng}>}%ExpressionSTR)) >= 0,''green'',''red'')' - WHEN MeasureDirectionCD = 'Down' - THEN 'IF( @(=ONLY({<%MeasureID={'+CAST(MeasureID AS VARCHAR)+'},%ElementCD={'+ElementCD+'},%ExpressionCD={BaseChng}>}%ExpressionSTR)) > 0,''red'',''green'')' - END AS ExpressionSTR - FROM LinkMeasureElementCTE - - UNION - SELECT - MeasureID - ,ElementCD - ,'PeriodChng_Color' AS ExpressionCD - ,CASE - WHEN MeasureDirectionCD = 'n/a' - THEN 'none' - WHEN MeasureDirectionCD = 'Up' - THEN 'IF( @(=ONLY({<%MeasureID={'+CAST(MeasureID AS VARCHAR)+'},%ElementCD={'+ElementCD+'},%ExpressionCD={PeriodChng}>}%ExpressionSTR)) >= 0,''green'',''red'')' - WHEN MeasureDirectionCD = 'Down' - THEN 'IF( @(=ONLY({<%MeasureID={'+CAST(MeasureID AS VARCHAR)+'},%ElementCD={'+ElementCD+'},%ExpressionCD={PeriodChng}>}%ExpressionSTR)) > 0,''red'',''green'')' - END AS ExpressionSTR - FROM LinkMeasureElementCTE -) - -/** union all the magic together **/ -SELECT * -FROM StandardExpressionCTE -UNION ALL -SELECT * -FROM InfoExpressionCTE"; + SELECT * FROM StandardExpressionCTE"; var parser = new Parser(false, true, true); var results = parser.GetTables(sql); - Assert.AreEqual(14, results.SelectMany(r => r.Columns).Count()); + Assert.AreEqual(2, results.SelectMany(r => r.Columns).Count()); } [Test] @@ -435,7 +272,7 @@ ICDDiagnosisCD FOR ICDDiagnosisCodes IN ( var parser = new Parser(); var results = parser.GetTables(sql); - Assert.AreEqual(2, results.SelectMany(r => r.Columns).Count()); + Assert.AreEqual(8, results.SelectMany(r => r.Columns).Count()); } [Test] @@ -548,5 +385,57 @@ public void DatePartUnitIgnored() Assert.AreEqual(1, results.SelectMany(r => r.Columns).Count()); } - } + + [Test] + public void UnpivotColumns() + { + var sql = @"SELECT pv.[Value] + FROM Table1 AS t1 + UNPIVOT + ( + [Value] + FOR [Observed] IN([Column1],[Column2]) + ) AS pv"; + var parser = new Parser(); + var results = parser.GetTables(sql); + + Assert.AreEqual(2, results.SelectMany(r => r.Columns).Count()); + } + + [Test] + public void UnpivotWithJoinColumns() + { + var sql = @"SELECT unp.ColumnNames, unp.ColumnValues, unp.Value1, t2.Category1, t2.Value1 AS t2Value1 FROM ( + SELECT + t1.Item1 + ,t1.Item2 + ,t1.Value1 + FROM table1 AS t1 + ) AS a + UNPIVOT (ColumnValues FOR ColumnNames IN (Item1,Item2)) AS unp + INNER JOIN table2 AS t2 ON t2.[ItemNM] = unp.[ColumnValues]"; + var parser = new Parser(); + var results = parser.GetTables(sql); + + Assert.AreEqual(6, results.SelectMany(r => r.Columns).Count()); + } + + [Test] + public void UnpivotWithUnAliasedColumns() + { + var sql = @"SELECT ColumnNames, ColumnValues, unp.Value1, t2.Category1, t2.Value1 AS t2Value1 FROM ( + SELECT + t1.Item1 + ,t1.Item2 + ,t1.Value1 + FROM table1 AS t1 + ) AS a + UNPIVOT (ColumnValues FOR ColumnNames IN (Item1,Item2)) AS unp + INNER JOIN table2 AS t2 ON t2.[ItemNM] = unp.[ColumnValues]"; + var parser = new Parser(); + var results = parser.GetTables(sql); + + Assert.AreEqual(6, results.SelectMany(r => r.Columns).Count()); + } + } }