diff --git a/src/Microsoft.Kusto.ServiceLayer/DataSource/Intellisense/ShowMaterializedViewSchemaResult.cs b/src/Microsoft.Kusto.ServiceLayer/DataSource/Intellisense/ShowMaterializedViewSchemaResult.cs new file mode 100644 index 0000000000..f2f4f8e189 --- /dev/null +++ b/src/Microsoft.Kusto.ServiceLayer/DataSource/Intellisense/ShowMaterializedViewSchemaResult.cs @@ -0,0 +1,16 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// + +namespace Microsoft.Kusto.ServiceLayer.DataSource.Intellisense +{ + public class ShowMaterializedViewSchemaResult + { + public string TableName; + public string Schema; + public string DatabaseName; + public string Folder; + public string DocString; + } +} diff --git a/src/Microsoft.Kusto.ServiceLayer/DataSource/Intellisense/ShowMaterializedViewsResult.cs b/src/Microsoft.Kusto.ServiceLayer/DataSource/Intellisense/ShowMaterializedViewsResult.cs new file mode 100644 index 0000000000..ae3154f095 --- /dev/null +++ b/src/Microsoft.Kusto.ServiceLayer/DataSource/Intellisense/ShowMaterializedViewsResult.cs @@ -0,0 +1,24 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// + +namespace Microsoft.Kusto.ServiceLayer.DataSource.Intellisense +{ + public class ShowMaterializedViewsResult + { + public string Name; + public string SourceTable; + public string Query; + public string MaterializedTo; + public string LastRun; + public string LastRunResult; + public bool IsHealthy; + public bool IsEnabled; + public string Folder; + public string DocString; + public bool AutoUpdateSchema; + public string EffectiveDateTime; + public string LookBack; + } +} diff --git a/src/Microsoft.Kusto.ServiceLayer/DataSource/Kusto/KustoDataSource.cs b/src/Microsoft.Kusto.ServiceLayer/DataSource/Kusto/KustoDataSource.cs index c49f222451..d82f278b99 100644 --- a/src/Microsoft.Kusto.ServiceLayer/DataSource/Kusto/KustoDataSource.cs +++ b/src/Microsoft.Kusto.ServiceLayer/DataSource/Kusto/KustoDataSource.cs @@ -46,16 +46,21 @@ public class KustoDataSource : DataSourceBase /// private ConcurrentDictionary> _tableMetadata = new ConcurrentDictionary>(); + /// + /// List of materialized views per database. Key - Parent Folder or Database Urn + /// + private ConcurrentDictionary> _materializedViewMetadata = new ConcurrentDictionary>(); + /// /// List of columns per table. Key - DatabaseName.TableName /// private ConcurrentDictionary> _columnMetadata = new ConcurrentDictionary>(); - + /// /// List of tables per database. Key - Parent Folder or Database Urn /// private ConcurrentDictionary> _folderMetadata = new ConcurrentDictionary>(); - + /// /// List of functions per database. Key - Parent Folder or Database Urn /// @@ -70,7 +75,7 @@ public override string DatabaseName public override string ClusterName => _kustoClient.ClusterName; // Some clusters have this signature. Queries might slightly differ for Aria - private const string AriaProxyURL = "kusto.aria.microsoft.com"; + private const string AriaProxyURL = "kusto.aria.microsoft.com"; /// /// The database schema query. Performance: ".show database schema" is more efficient than ".show schema", @@ -82,7 +87,7 @@ public override string DatabaseName /// The dashboard needs a list of all tables regardless of the folder structure of the table. The /// tables are stored with the key in the following format: OnlyTables.ClusterName.DatabaseName /// - private const string DatabaseKeyPrefix = "OnlyTables"; + private const string DatabaseKeyPrefix = "OnlyTablesAndViews"; /// /// Prevents a default instance of the class from being created. @@ -95,7 +100,7 @@ public KustoDataSource(IKustoClient kustoClient, IntellisenseClientBase intellis ValidationUtils.IsTrue(Exists().Result, $"Unable to connect. ClusterName = {ClusterName}, DatabaseName = {DatabaseName}"); } - + /// /// Disposes resources. /// @@ -134,10 +139,10 @@ public override async Task Exists() if (ClusterName.Contains(AriaProxyURL, StringComparison.OrdinalIgnoreCase)) { - var result = await ExecuteScalarQueryAsync(".show databases | take 1 | project DatabaseName", source.Token); + var result = await ExecuteScalarQueryAsync(".show databases | take 1 | project DatabaseName", source.Token); return !string.IsNullOrWhiteSpace(result); } - + var count = await ExecuteScalarQueryAsync(".show databases | count", source.Token); return count >= 0; } @@ -148,7 +153,7 @@ public override async Task Exists() } #endregion - + #region IDataSource private DiagnosticsInfo GetClusterDiagnostics() @@ -157,11 +162,11 @@ private DiagnosticsInfo GetClusterDiagnostics() { return new DiagnosticsInfo(); } - + var source = new CancellationTokenSource(); var clusterDiagnostics = new DiagnosticsInfo(); - var query = ".show diagnostics | extend Passed= (IsHealthy) and not(IsScaleOutRequired) | extend Summary = strcat('Cluster is ', iif(Passed, '', 'NOT'), 'healthy.'),Details=pack('MachinesTotal', MachinesTotal, 'DiskCacheCapacity', round(ClusterDataCapacityFactor,1)) | project Action = 'Cluster Diagnostics', Category='Info', Summary, Details;"; + var query = ".show diagnostics | extend Passed= (IsHealthy) and not(IsScaleOutRequired) | extend Summary = strcat('Cluster is ', iif(Passed, '', 'NOT'), 'healthy.'),Details=pack('MachinesTotal', MachinesTotal, 'DiskCacheCapacity', round(ClusterDataCapacityFactor,1)) | project Action = 'Cluster Diagnostics', Category='Info', Summary, Details;"; using (var reader = _kustoClient.ExecuteQuery(query, source.Token)) { while (reader.Read()) @@ -193,7 +198,7 @@ private void SetDatabaseMetadata(bool includeSizeDetails) { includeSizeDetails = false; } - + CancellationTokenSource source = new CancellationTokenSource(); CancellationToken token = source.Token; @@ -215,7 +220,7 @@ private void SetDatabaseMetadata(bool includeSizeDetails) MetadataTypeName = DataSourceMetadataType.Database.ToString(), SizeInMB = includeSizeDetails ? row["sum_OriginalSize"].ToString() : "", Name = row["DatabaseName"].ToString(), - PrettyName = includeSizeDetails ? row["DatabaseName"].ToString(): (string.IsNullOrEmpty(row["PrettyName"]?.ToString()) ? row["DatabaseName"].ToString() : row["PrettyName"].ToString()), + PrettyName = includeSizeDetails ? row["DatabaseName"].ToString() : (string.IsNullOrEmpty(row["PrettyName"]?.ToString()) ? row["DatabaseName"].ToString() : row["PrettyName"].ToString()), Urn = $"{ClusterName}.{row["DatabaseName"]}" }) .Materialize() @@ -228,7 +233,7 @@ public override bool Exists(DataSourceObjectMetadata objectMetadata) { ValidationUtils.IsNotNull(objectMetadata, "Need a datasource object"); - switch(objectMetadata.MetadataType) + switch (objectMetadata.MetadataType) { case DataSourceMetadataType.Database: return DatabaseExists(objectMetadata.Name).Result; default: throw new ArgumentException($"Unexpected type {objectMetadata.MetadataType}."); @@ -261,7 +266,7 @@ public override void UpdateDatabase(string databaseName) _kustoClient.UpdateDatabase(parsedDatabase); _intellisenseClient.UpdateDatabase(parsedDatabase); } - + /// /// Clears everything /// @@ -274,7 +279,8 @@ public override void Refresh(bool includeDatabase) _databaseMetadata = null; } _tableMetadata = new ConcurrentDictionary>(); - _columnMetadata = new ConcurrentDictionary>(); + _materializedViewMetadata = new ConcurrentDictionary>(); + _columnMetadata = new ConcurrentDictionary>(); _folderMetadata = new ConcurrentDictionary>(); _functionMetadata = new ConcurrentDictionary>(); } @@ -284,17 +290,18 @@ public override void Refresh(DataSourceObjectMetadata objectMetadata) { ValidationUtils.IsNotNull(objectMetadata, nameof(objectMetadata)); - switch(objectMetadata.MetadataType) + switch (objectMetadata.MetadataType) { case DataSourceMetadataType.Cluster: Refresh(true); SetDatabaseMetadata(false); break; - + case DataSourceMetadataType.Database: Refresh(false); LoadTableSchema(objectMetadata); LoadFunctionSchema(objectMetadata); + LoadMaterializedViewSchema(objectMetadata); break; case DataSourceMetadataType.Table: @@ -303,13 +310,20 @@ public override void Refresh(DataSourceObjectMetadata objectMetadata) SetTableSchema(table); break; + case DataSourceMetadataType.MaterializedView: + var materializedView = objectMetadata as TableMetadata; + _columnMetadata.TryRemove(GenerateMetadataKey(materializedView.DatabaseName, materializedView.Name), out _); + SetMaterializedViewSchema(materializedView); + break; + case DataSourceMetadataType.Folder: Refresh(false); var folder = objectMetadata as FolderMetadata; LoadTableSchema(folder.ParentMetadata); LoadFunctionSchema(folder.ParentMetadata); + LoadMaterializedViewSchema(folder.ParentMetadata); break; - + default: throw new ArgumentException($"Unexpected type {objectMetadata.MetadataType}."); } @@ -328,13 +342,17 @@ public override IEnumerable GetChildObjects(DataSource case DataSourceMetadataType.Database: // show folders, tables, and functions return includeSizeDetails - ? GetTablesForDashboard(objectMetadata) + ? GetTablesAndViewsForDashboard(objectMetadata) : GetDatabaseSchema(objectMetadata); case DataSourceMetadataType.Table: // show columns var table = objectMetadata as TableMetadata; return GetTableSchema(table); + case DataSourceMetadataType.MaterializedView: // show materialized view columns + var materializedView = objectMetadata as TableMetadata; + return GetMaterializedViewSchema(materializedView); + case DataSourceMetadataType.Folder: // show subfolders, functions, and tables var folder = objectMetadata as FolderMetadata; return GetAllMetadata(folder.Urn); @@ -349,7 +367,7 @@ public override DiagnosticsInfo GetDiagnostics(DataSourceObjectMetadata objectMe ValidationUtils.IsNotNull(objectMetadata, nameof(objectMetadata)); // Add more cases when required. - switch(objectMetadata.MetadataType) + switch (objectMetadata.MetadataType) { case DataSourceMetadataType.Cluster: return GetClusterDiagnostics(); @@ -376,7 +394,7 @@ internal async Task DatabaseExists(string databaseName) return false; } } - + /// private IEnumerable GetDatabaseSchema(DataSourceObjectMetadata objectMetadata) { @@ -384,7 +402,7 @@ private IEnumerable GetDatabaseSchema(DataSourceObject ValidationUtils.IsTrue(DatabaseExists(objectMetadata.Name).Result, $"Database '{objectMetadata}' does not exist."); var allMetadata = GetAllMetadata(objectMetadata.Urn); - + // if the records have already been loaded them return them if (allMetadata.Any()) { @@ -393,20 +411,28 @@ private IEnumerable GetDatabaseSchema(DataSourceObject LoadTableSchema(objectMetadata); LoadFunctionSchema(objectMetadata); - + LoadMaterializedViewSchema(objectMetadata); + return GetAllMetadata(objectMetadata.Urn); } - private IEnumerable GetTablesForDashboard(DataSourceObjectMetadata objectMetadata) + private IEnumerable GetTablesAndViewsForDashboard(DataSourceObjectMetadata objectMetadata) { string newKey = $"{DatabaseKeyPrefix}.{objectMetadata.Urn}"; if (!_tableMetadata.ContainsKey(newKey) || !_tableMetadata[newKey].Any()) { - LoadTableSchema(objectMetadata); + LoadTableSchema(objectMetadata); } - - return _tableMetadata[newKey].OrderBy(x => x.PrettyName, StringComparer.OrdinalIgnoreCase); + + if (!_materializedViewMetadata.ContainsKey(newKey) || !_materializedViewMetadata[newKey].Any()) + { + LoadMaterializedViewSchema(objectMetadata); + } + + return _tableMetadata[newKey] + .OrderBy(x => x.PrettyName, StringComparer.OrdinalIgnoreCase) + .Concat(_materializedViewMetadata[newKey].OrderBy(x => x.PrettyName)); } private IEnumerable GetAllMetadata(string key) @@ -423,6 +449,11 @@ private IEnumerable GetAllMetadata(string key) returnList.AddRange(tableMetadata.OrderBy(x => x.PrettyName, StringComparer.OrdinalIgnoreCase)); } + if (_materializedViewMetadata.TryGetValue(key, out IEnumerable? materializedViewMetadata)) + { + returnList.AddRange(materializedViewMetadata.OrderBy(x => x.PrettyName, StringComparer.OrdinalIgnoreCase)); + } + if (_functionMetadata.TryGetValue(key, out IEnumerable? functionMetadata)) { returnList.AddRange(functionMetadata.OrderBy(x => x.PrettyName, StringComparer.OrdinalIgnoreCase)); @@ -467,6 +498,41 @@ private IEnumerable GetColumnInfos(string databaseName, string table } } + private IEnumerable GetMaterializedViewColumnInfos(string databaseName, string materializedViewName) + { + ValidationUtils.IsNotNullOrWhitespace(databaseName, nameof(databaseName)); + + CancellationTokenSource source = new CancellationTokenSource(); + CancellationToken token = source.Token; + + const string systemPrefix = "System."; + var query = new StringBuilder(string.Format(CultureInfo.InvariantCulture, ".show materialized-view [{0}] schema as json", + materializedViewName)); + query.Append(" | project TableName, Schema, Folder"); + + using (var reader = _kustoClient.ExecuteQuery(query.ToString(), token, databaseName)) + { + var result = reader.ToEnumerable() + .FirstOrDefault(); + if (result == null || result["Schema"] == null) + return Enumerable.Empty(); + + var columns = JsonConvert.DeserializeObject(result["Schema"].ToString()) + .OrderedColumns + .Select(column => new ColumnInfo + { + Table = result["TableName"]?.ToString(), + Name = column.Name?.ToString(), + DataType = column.Type?.ToString().TrimPrefix(systemPrefix), + Folder = result["Folder"]?.ToString() + }) + .Materialize() + .OrderBy(column => column.Name, StringComparer.Ordinal); // case-sensitive + + return columns; + } + } + private IEnumerable GetTableInfos(string databaseName) { CancellationTokenSource source = new CancellationTokenSource(); @@ -494,21 +560,63 @@ private void LoadTableSchema(DataSourceObjectMetadata databaseMetadata) { return; } - + var rootTableFolderKey = new StringBuilder($"{databaseMetadata.Urn}"); if (tableInfos.Any(x => !string.IsNullOrWhiteSpace(x.Folder))) { - // create Table folder to hold functions tables + // create Table folder to hold tables folders var tableFolder = MetadataFactory.CreateFolderMetadata(databaseMetadata, rootTableFolderKey.ToString(), "Tables"); - _folderMetadata.AddRange(rootTableFolderKey.ToString(), new List {tableFolder}); + _folderMetadata.AddRange(rootTableFolderKey.ToString(), new List { tableFolder }); rootTableFolderKey.Append($".{tableFolder.Name}"); - + SetFolderMetadataForTables(databaseMetadata, tableInfos, rootTableFolderKey.ToString()); } - + SetTableMetadata(databaseMetadata, tableInfos, rootTableFolderKey.ToString()); } + private IEnumerable GetMaterializedViewsInfos(string databaseName) + { + CancellationTokenSource source = new CancellationTokenSource(); + CancellationToken token = source.Token; + + string query = $".show materialized-views"; + + using (var reader = _kustoClient.ExecuteQuery(query, token, databaseName)) + { + return reader.ToEnumerable() + .Select(row => new TableInfo + { + TableName = row["Name"]?.ToString(), + Folder = row["Folder"]?.ToString() + }) + .Materialize(); + } + } + + private void LoadMaterializedViewSchema(DataSourceObjectMetadata databaseMetadata) + { + var materializedViewInfos = GetMaterializedViewsInfos(databaseMetadata.Name); + + if (!materializedViewInfos.Any()) + { + return; + } + + var rootMaterializedViewFolderKey = new StringBuilder($"{databaseMetadata.Urn}"); + if (materializedViewInfos.Any(x => !string.IsNullOrWhiteSpace(x.Folder))) + { + // create Materialized views folder to hold materialized views folders + var materializedViewFolder = MetadataFactory.CreateFolderMetadata(databaseMetadata, rootMaterializedViewFolderKey.ToString(), "Materialized views"); + _folderMetadata.AddRange(rootMaterializedViewFolderKey.ToString(), new List { materializedViewFolder }); + rootMaterializedViewFolderKey.Append($".{materializedViewFolder.Name}"); + + SetFolderMetadataForTables(databaseMetadata, materializedViewInfos, rootMaterializedViewFolderKey.ToString()); + } + + SetMaterializedViewMetadata(databaseMetadata, materializedViewInfos, rootMaterializedViewFolderKey.ToString()); + } + private IEnumerable GetTableSchema(TableMetadata tableMetadata) { var key = GenerateMetadataKey(tableMetadata.DatabaseName, tableMetadata.Name); @@ -516,10 +624,10 @@ private IEnumerable GetTableSchema(TableMetadata table { return metadata; } - + SetTableSchema(tableMetadata); - return _columnMetadata.TryGetValue(key, out IEnumerable? columnMetadata) + return _columnMetadata.TryGetValue(key, out IEnumerable? columnMetadata) ? columnMetadata : Enumerable.Empty(); } @@ -531,16 +639,42 @@ private void SetTableSchema(TableMetadata tableMetadata) { return; } - + SetColumnMetadata(tableMetadata.DatabaseName, tableMetadata.Name, columnInfos); } + private IEnumerable GetMaterializedViewSchema(TableMetadata materializedViewMetadata) + { + var key = GenerateMetadataKey(materializedViewMetadata.DatabaseName, materializedViewMetadata.Name); + if (_columnMetadata.TryGetValue(key, out IEnumerable? metadata)) + { + return metadata; + } + + SetMaterializedViewSchema(materializedViewMetadata); + + return _columnMetadata.TryGetValue(key, out IEnumerable? columnMetadata) + ? columnMetadata : Enumerable.Empty(); + } + + private void SetMaterializedViewSchema(TableMetadata materializedViewMetadata) + { + IEnumerable columnInfos = GetMaterializedViewColumnInfos(materializedViewMetadata.DatabaseName, materializedViewMetadata.Name); + + if (!columnInfos.Any()) + { + return; + } + + SetColumnMetadata(materializedViewMetadata.DatabaseName, materializedViewMetadata.Name, columnInfos); + } + private void SetFolderMetadataForTables(DataSourceObjectMetadata objectMetadata, IEnumerable tableInfos, string rootTableFolderKey) { var tablesByFolder = tableInfos .GroupBy(x => x.Folder, StringComparer.OrdinalIgnoreCase) .ToList(); - + var tableFolders = new List(); foreach (var columnGroup in tablesByFolder) @@ -570,7 +704,7 @@ private void LoadFunctionSchema(DataSourceObjectMetadata databaseMetadata) // create Functions folder to hold functions folders var rootFunctionFolderKey = $"{databaseMetadata.Urn}"; var rootFunctionFolder = MetadataFactory.CreateFolderMetadata(databaseMetadata, rootFunctionFolderKey, "Functions"); - _folderMetadata.AddRange(rootFunctionFolderKey, new List {rootFunctionFolder}); + _folderMetadata.AddRange(rootFunctionFolderKey, new List { rootFunctionFolder }); // create each folder to hold functions var functionsGroupByFolder = functionInfos @@ -579,7 +713,7 @@ private void LoadFunctionSchema(DataSourceObjectMetadata databaseMetadata) if (functionsGroupByFolder.Any()) { - SetFolderMetadataForFunctions(databaseMetadata, functionsGroupByFolder, rootFunctionFolder); + SetFolderMetadataForFunctions(databaseMetadata, functionsGroupByFolder, rootFunctionFolder); } SetFunctionMetadata(databaseMetadata.Name, rootFunctionFolder.Name, functionsGroupByFolder); @@ -628,8 +762,8 @@ private void SetColumnMetadata(string databaseName, string tableName, IEnumerabl && !string.IsNullOrWhiteSpace(row.Name) && !string.IsNullOrWhiteSpace(row.DataType)); - var columnMetadatas = new SortedList(); - + var columnMetadata = new SortedList(); + foreach (ColumnInfo columnInfo in columns) { var column = new ColumnMetadata @@ -645,10 +779,58 @@ private void SetColumnMetadata(string databaseName, string tableName, IEnumerabl DataType = columnInfo.DataType }; - columnMetadatas.Add(column.PrettyName, column); + columnMetadata.Add(column.PrettyName, column); + } + + _columnMetadata[GenerateMetadataKey(databaseName, tableName)] = columnMetadata.Values; + } + + private void SetMaterializedViewMetadata(DataSourceObjectMetadata databaseName, IEnumerable materializedViewInfos, string rootTableFolderKey) + { + var materializedViewFolders = new Dictionary> + { + {$"{DatabaseKeyPrefix}.{databaseName.Urn}", new List()} + }; + + foreach (var materializedView in materializedViewInfos) + { + var tableKey = new StringBuilder(rootTableFolderKey); + + if (!string.IsNullOrWhiteSpace(materializedView.Folder)) + { + tableKey.Append($".{materializedView.Folder}"); + } + + var materializedViewMetadata = new TableMetadata + { + ClusterName = ClusterName, + DatabaseName = databaseName.Name, + MetadataType = DataSourceMetadataType.MaterializedView, + MetadataTypeName = DataSourceMetadataType.MaterializedView.ToString(), + Name = materializedView.TableName, + PrettyName = materializedView.TableName, + Folder = materializedView.Folder, + Urn = $"{tableKey}.{materializedView.TableName}" + }; + + if (materializedViewFolders.TryGetValue(tableKey.ToString(), out List? tableMetadataList)) + { + tableMetadataList.Add(materializedViewMetadata); + } + else + { + materializedViewFolders[tableKey.ToString()] = new List { materializedViewMetadata }; + } + + // keep a list of all tables for the database + // this is used for the dashboard + materializedViewFolders[$"{DatabaseKeyPrefix}.{databaseName.Urn}"].Add(materializedViewMetadata); } - _columnMetadata[GenerateMetadataKey(databaseName, tableName)] = columnMetadatas.Values; + foreach (var table in materializedViewFolders) + { + _materializedViewMetadata.AddRange(table.Key, table.Value); + } } private void SetTableMetadata(DataSourceObjectMetadata databaseName, IEnumerable tableInfos, string rootTableFolderKey) @@ -657,7 +839,7 @@ private void SetTableMetadata(DataSourceObjectMetadata databaseName, IEnumerable { {$"{DatabaseKeyPrefix}.{databaseName.Urn}", new List()} }; - + foreach (var table in tableInfos) { var tableKey = new StringBuilder(rootTableFolderKey); @@ -685,14 +867,14 @@ private void SetTableMetadata(DataSourceObjectMetadata databaseName, IEnumerable } else { - tableFolders[tableKey.ToString()] = new List{tableMetadata}; + tableFolders[tableKey.ToString()] = new List { tableMetadata }; } - + // keep a list of all tables for the database // this is used for the dashboard tableFolders[$"{DatabaseKeyPrefix}.{databaseName.Urn}"].Add(tableMetadata); } - + foreach (var table in tableFolders) { _tableMetadata.AddRange(table.Key, table.Value); @@ -705,22 +887,22 @@ private void SetFunctionMetadata(string databaseName, string rootFunctionFolderK foreach (var functionGroup in functionGroupByFolder) { var stringBuilder = new StringBuilder(rootFunctionFolderKey); - + if (!string.IsNullOrWhiteSpace(functionGroup.Key)) { stringBuilder.Append("."); - + // folders are in the following format: folder1/folder2/folder3/folder4 var folderKey = functionGroup.Key .Replace(@"\", ".") .Replace(@"/", "."); - + stringBuilder.Append(folderKey); } var functionKey = $"{ClusterName}.{databaseName}.{stringBuilder}"; var functions = new List(); - + foreach (FunctionInfo functionInfo in functionGroup) { var function = new FunctionMetadata @@ -789,12 +971,12 @@ private FunctionInfo GetFunctionInfo(string functionName) public override string GenerateAlterFunctionScript(string functionName) { var functionInfo = GetFunctionInfo(functionName); - + if (functionInfo == null) { return string.Empty; } - + var alterCommand = new StringBuilder(); alterCommand.Append(".alter function with "); @@ -808,12 +990,12 @@ public override string GenerateAlterFunctionScript(string functionName) public override string GenerateExecuteFunctionScript(string functionName) { var functionInfo = GetFunctionInfo(functionName); - - return functionInfo == null - ? string.Empty + + return functionInfo == null + ? string.Empty : $"{functionInfo.Name}{functionInfo.Parameters}"; } - + private string GenerateMetadataKey(string databaseName, string objectName) { return string.IsNullOrWhiteSpace(objectName) ? databaseName : $"{databaseName}.{objectName}"; @@ -866,9 +1048,9 @@ public override ListDatabasesResponse GetDatabases(string serverName, bool inclu }; } - return new ListDatabasesResponse();; + return new ListDatabasesResponse(); ; } - + public override DatabaseInfo GetDatabaseInfo(string serverName, string databaseName) { DataSourceObjectMetadata objectMetadata = MetadataFactory.CreateClusterMetadata(serverName); diff --git a/src/Microsoft.Kusto.ServiceLayer/DataSource/Kusto/KustoIntellisenseClient.cs b/src/Microsoft.Kusto.ServiceLayer/DataSource/Kusto/KustoIntellisenseClient.cs index 7afeb25b6a..4929f308bd 100644 --- a/src/Microsoft.Kusto.ServiceLayer/DataSource/Kusto/KustoIntellisenseClient.cs +++ b/src/Microsoft.Kusto.ServiceLayer/DataSource/Kusto/KustoIntellisenseClient.cs @@ -4,6 +4,7 @@ // using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using System.Threading; @@ -24,16 +25,17 @@ public KustoIntellisenseClient(IKustoClient kustoClient) _kustoClient = kustoClient; schemaState = LoadSchemaState(kustoClient.DatabaseName, kustoClient.ClusterName); } - + public override void UpdateDatabase(string databaseName) { schemaState = LoadSchemaState(databaseName, _kustoClient.ClusterName); } - + private GlobalState LoadSchemaState(string databaseName, string clusterName) { IEnumerable tableSchemas = Enumerable.Empty(); IEnumerable functionSchemas = Enumerable.Empty(); + var materializedViewSchemas = new ConcurrentBag(); if (!string.IsNullOrWhiteSpace(databaseName)) { @@ -46,26 +48,43 @@ private GlobalState LoadSchemaState(string databaseName, string clusterName) () => { functionSchemas = _kustoClient.ExecuteQueryAsync(".show functions", source.Token, databaseName).Result; + }, + () => + { + var materializedViews = _kustoClient.ExecuteQueryAsync(".show materialized-views", source.Token, databaseName).Result; + Parallel.ForEach(materializedViews, materializedView => + { + var materializedViewSchema = _kustoClient + .ExecuteQueryAsync( + $".show materialized-view {materializedView.Name} cslschema", source.Token, + databaseName).Result + .FirstOrDefault(); + + if (materializedViewSchema != null) + { + materializedViewSchemas.Add(materializedViewSchema); + } + }); }); } - return AddOrUpdateDatabase(tableSchemas, functionSchemas, GlobalState.Default, databaseName, + return AddOrUpdateDatabase(tableSchemas, functionSchemas, materializedViewSchemas, GlobalState.Default, databaseName, clusterName); } - + /// /// Loads the schema for the specified database and returns a new with the database added or updated. /// private GlobalState AddOrUpdateDatabase(IEnumerable tableSchemas, - IEnumerable functionSchemas, GlobalState globals, - string databaseName, string clusterName) + IEnumerable functionSchemas, IEnumerable materializedViewSchemas, + GlobalState globals, string databaseName, string clusterName) { // try and show error from here. DatabaseSymbol databaseSymbol = null; if (databaseName != null) { - databaseSymbol = LoadDatabase(tableSchemas, functionSchemas, databaseName); + databaseSymbol = LoadDatabase(tableSchemas, functionSchemas, materializedViewSchemas, databaseName); } if (databaseSymbol == null) @@ -76,7 +95,7 @@ private GlobalState AddOrUpdateDatabase(IEnumerable ta var cluster = globals.GetCluster(clusterName); if (cluster == null) { - cluster = new ClusterSymbol(clusterName, new[] {databaseSymbol}, isOpen: true); + cluster = new ClusterSymbol(clusterName, new[] { databaseSymbol }, isOpen: true); globals = globals.AddOrUpdateCluster(cluster); } else @@ -87,12 +106,12 @@ private GlobalState AddOrUpdateDatabase(IEnumerable ta return globals.WithCluster(cluster).WithDatabase(databaseSymbol); } - + /// /// Loads the schema for the specified database into a . /// private DatabaseSymbol LoadDatabase(IEnumerable tableSchemas, - IEnumerable functionSchemas, + IEnumerable functionSchemas, IEnumerable materializedViewSchemas, string databaseName) { if (tableSchemas == null) @@ -112,21 +131,34 @@ private DatabaseSymbol LoadDatabase(IEnumerable tableS members.Add(tableSymbol); } - if (functionSchemas == null) + if (functionSchemas != null) { - return null; + foreach (var fun in functionSchemas) + { + var parameters = TranslateParameters(fun.Parameters); + var functionSymbol = new FunctionSymbol(fun.Name, fun.Body, parameters); + members.Add(functionSymbol); + } } - foreach (var fun in functionSchemas) + if (materializedViewSchemas != null) { - var parameters = TranslateParameters(fun.Parameters); - var functionSymbol = new FunctionSymbol(fun.Name, fun.Body, parameters); - members.Add(functionSymbol); + foreach (var view in materializedViewSchemas) + { + var columns = view.Schema.Split(',') + .Select(col => + { + var nameType = col.Split(':'); + return new ColumnSymbol(nameType[0], ScalarTypes.GetSymbol(nameType[1])); + }); + var viewSymbol = new TableSymbol(view.TableName, columns); + members.Add(viewSymbol); + } } return new DatabaseSymbol(databaseName, members); } - + /// /// Convert CLR type name into a Kusto scalar type. /// @@ -200,7 +232,7 @@ private ScalarSymbol GetKustoType(string clrTypeName) throw new InvalidOperationException($"Unhandled clr type: {clrTypeName}"); } } - + /// /// Translate Kusto parameter list declaration into into list of instances. /// @@ -226,7 +258,7 @@ private IReadOnlyList TranslateParameters(string parameters) var query = "let fn = " + parameters + " { };"; var code = KustoCode.ParseAndAnalyze(query); var let = code.Syntax.GetFirstDescendant(); - + FunctionSymbol function = let.Name.ReferencedSymbol is VariableSymbol variable ? variable.Type as FunctionSymbol : let.Name.ReferencedSymbol as FunctionSymbol; diff --git a/src/Microsoft.Kusto.ServiceLayer/DataSource/Metadata/DataSourceMetadataType.cs b/src/Microsoft.Kusto.ServiceLayer/DataSource/Metadata/DataSourceMetadataType.cs index dcf4fa3317..a6d385eeb4 100644 --- a/src/Microsoft.Kusto.ServiceLayer/DataSource/Metadata/DataSourceMetadataType.cs +++ b/src/Microsoft.Kusto.ServiceLayer/DataSource/Metadata/DataSourceMetadataType.cs @@ -15,6 +15,7 @@ public enum DataSourceMetadataType Table = 2, Column = 3, Function = 4, - Folder = 5 + Folder = 5, + MaterializedView = 6 } } \ No newline at end of file diff --git a/src/Microsoft.Kusto.ServiceLayer/DataSource/Metadata/MetadataFactory.cs b/src/Microsoft.Kusto.ServiceLayer/DataSource/Metadata/MetadataFactory.cs index a30adeb17c..4fb740fa4c 100644 --- a/src/Microsoft.Kusto.ServiceLayer/DataSource/Metadata/MetadataFactory.cs +++ b/src/Microsoft.Kusto.ServiceLayer/DataSource/Metadata/MetadataFactory.cs @@ -109,7 +109,9 @@ public static List ConvertToObjectMetadata(IEnumerable OrderedColumns { get; set; } + } + + public class MaterializedViewColumnInfo + { + public string Name { get; set; } + + public string Type { get; set; } + + public string CslType { get; set; } + } +} diff --git a/src/Microsoft.Kusto.ServiceLayer/ObjectExplorer/Nodes/TreeNode.cs b/src/Microsoft.Kusto.ServiceLayer/ObjectExplorer/Nodes/TreeNode.cs index 56566a55fc..d7a48a1435 100644 --- a/src/Microsoft.Kusto.ServiceLayer/ObjectExplorer/Nodes/TreeNode.cs +++ b/src/Microsoft.Kusto.ServiceLayer/ObjectExplorer/Nodes/TreeNode.cs @@ -8,8 +8,8 @@ using System.Collections.ObjectModel; using System.Diagnostics; using System.Globalization; -using System.Threading; using System.Linq; +using System.Threading; using Microsoft.Kusto.ServiceLayer.DataSource; using Microsoft.Kusto.ServiceLayer.DataSource.Metadata; using Microsoft.Kusto.ServiceLayer.ObjectExplorer.Contracts; @@ -72,7 +72,8 @@ public object BuildingMetadataLock /// /// The name of this object as included in its node path /// - public string NodePathName { + public string NodePathName + { get { if (string.IsNullOrEmpty(nodePathName)) @@ -123,10 +124,11 @@ public string NodePathName { /// for many nodes such as the server, the display label will be different /// to the value. /// - protected string Label { + protected string Label + { get { - if(label == null) + if (label == null) { return NodeValue; } @@ -166,7 +168,7 @@ public TreeNode Parent nodePath = null; } } - + /// /// Path identifying this node: for example a table will be at ["server", "database", "tables", "tableName"]. /// This enables rapid navigation of the tree without the need for a global registry of elements. @@ -193,7 +195,7 @@ private void GenerateNodePath() return false; } // Otherwise add this value to the beginning of the path and keep iterating up - path = string.Format(CultureInfo.InvariantCulture, + path = string.Format(CultureInfo.InvariantCulture, "{0}{1}{2}", node.NodePathName, string.IsNullOrEmpty(path) ? "" : PathPartSeperator.ToString(), path); return true; }); @@ -311,7 +313,7 @@ protected void PopulateChildren(bool refresh, string name, CancellationToken can Debug.Assert(IsAlwaysLeaf == false); QueryContext context = this.GetContextAs(); - + if (children.IsPopulating || context == null) { return; @@ -332,7 +334,7 @@ protected void PopulateChildren(bool refresh, string name, CancellationToken can children.Add(item); item.Parent = this; } - } + } } catch (Exception ex) { @@ -386,7 +388,7 @@ private List OnExpandPopulateNonFolders(TreeNode parent, bool refresh, { parent.DataSource.Refresh(parent.ObjectMetadata); } - + objectMetadataList = parent.DataSource.GetChildObjects(parent.ObjectMetadata); } @@ -423,34 +425,44 @@ private List OnExpandPopulateNonFolders(TreeNode parent, bool refresh, /// /// /// - + private TreeNode CreateChild(TreeNode parent, DataSourceObjectMetadata childMetadata) { ValidationUtils.IsNotNull(parent, nameof(parent)); ValidationUtils.IsNotNull(childMetadata, nameof(childMetadata)); - switch(childMetadata.MetadataType) + switch (childMetadata.MetadataType) { - case DataSourceMetadataType.Database: - return new DataSourceTreeNode(parent.DataSource, childMetadata) { + case DataSourceMetadataType.Database: + return new DataSourceTreeNode(parent.DataSource, childMetadata) + { Parent = parent as ServerNode, NodeType = "Database", - NodeTypeId = NodeTypes.Database + NodeTypeId = NodeTypes.Database }; case DataSourceMetadataType.Table: - return new DataSourceTreeNode(parent.DataSource, childMetadata) { + return new DataSourceTreeNode(parent.DataSource, childMetadata) + { NodeType = "Table", - NodeTypeId = NodeTypes.Table + NodeTypeId = NodeTypes.Table + }; + + case DataSourceMetadataType.MaterializedView: + return new DataSourceTreeNode(parent.DataSource, childMetadata) + { + NodeType = "View", + NodeTypeId = NodeTypes.View }; case DataSourceMetadataType.Column: - return new DataSourceTreeNode(parent.DataSource, childMetadata) { + return new DataSourceTreeNode(parent.DataSource, childMetadata) + { IsAlwaysLeaf = true, NodeType = "Column", SortPriority = DataSourceTreeNode.NextSortPriority }; - + case DataSourceMetadataType.Folder: return new DataSourceTreeNode(parent.DataSource, childMetadata) { @@ -458,7 +470,7 @@ private TreeNode CreateChild(TreeNode parent, DataSourceObjectMetadata childMeta NodeType = "Folder", NodeTypeId = NodeTypes.Folder }; - + case DataSourceMetadataType.Function: return new DataSourceTreeNode(parent.DataSource, childMetadata) { @@ -495,7 +507,7 @@ protected virtual int CompareSamePriorities(TreeNode thisItem, TreeNode otherIte { return string.Compare(thisItem.NodeValue, otherItem.NodeValue, StringComparison.OrdinalIgnoreCase); } - + public int CompareTo(TreeNode other) {