Create and execute query to return tables and interpolated names for Autoentities

src/Core/Resolvers/IQueryBuilder.cs

@@ -90,5 +90,7 @@ public interface IQueryBuilder
         /// DB Connection Param.
         /// </summary>
         public string QuoteTableNameAsDBConnectionParam(string param);
+
+        public string BuildGetAutoentitiesQuery(string include, string exclude, string namePattern) => throw new NotSupportedException($"{GetType().Name} does not support Autoentities yet.");
     }
 }

src/Core/Resolvers/MsSqlQueryBuilder.cs

@@ -506,7 +506,7 @@ public string BuildStoredProcedureResultDetailsQuery(string databaseObjectName)
         /// 2. are computed based on other columns,
         /// are considered as read only columns. The query combines both the types of read-only columns and returns the list.
         /// </summary>
-        /// <param name="schemaOrDatabaseParamName">Param name of the schema/database.</param>
+        /// <param name="schemaParamName">Param name of the schema/database.</param>
         /// <param name="tableParamName">Param name of the table.</param>
         /// <returns></returns>
         public string BuildQueryToGetReadOnlyColumns(string schemaParamName, string tableParamName)
@@ -560,5 +560,39 @@ protected override string BuildPredicates(SqlQueryStructure structure)
             // contains LIKE and add the ESCAPE clause accordingly.
             return AddEscapeToLikeClauses(predicates);
         }
+
+        /// <summary>
+        /// Builds the query used to get the list of tables with the SQL LIKE
+        /// syntax that will be transformed into entities.
+        /// NOTE: Currently this query only returns Tables, support for Views will come later.
+        /// </summary>
+        /// <param name="include">Pattern for tables that will be included.</param>
+        /// <param name="exclude">Pattern for tables that will be excluded.</param>
+        /// <param name="namePattern">Pattern for naming the entities.</param>
+        public string BuildGetAutoentitiesQuery(string include, string exclude, string namePattern)
+        {
+            string query = $"DECLARE @include_pattern NVARCHAR(MAX) = '{include}'; DECLARE @exclude_pattern NVARCHAR(MAX) = '{exclude}'; DECLARE @name_pattern NVARCHAR(255) = '{namePattern}'; " +
+                "DECLARE @exclude_invalid_types BIT = 1; SET NOCOUNT ON; WITH include_patterns AS ( SELECT LTRIM(RTRIM(value)) AS pattern " +
+                "FROM STRING_SPLIT(ISNULL(@include_pattern, N''), N',') WHERE LTRIM(RTRIM(value)) <> N'' ), " +
+                "exclude_patterns AS ( SELECT LTRIM(RTRIM(value)) AS pattern FROM STRING_SPLIT(ISNULL(@exclude_pattern, N''), N',') " +
+                "WHERE LTRIM(RTRIM(value)) <> N'' ), all_tables AS ( SELECT s.name AS schema_name, t.name AS object_name, s.name + N'.' " +
+                "+ t.name AS full_name, N'table' AS object_type, t.object_id FROM sys.tables AS t JOIN sys.schemas AS s ON t.schema_id = s.schema_id " +
+                "WHERE EXISTS ( SELECT 1 FROM sys.key_constraints AS kc WHERE kc.parent_object_id = t.object_id AND kc.type = 'PK' ) ), eligible_tables AS " +
+                "( SELECT o.schema_name, o.object_name, o.full_name, o.object_type, o.object_id, CASE WHEN so.is_ms_shipped = 1 THEN 1 WHEN o.schema_name IN " +
+                "(N'sys', N'INFORMATION_SCHEMA') THEN 1 WHEN o.object_name IN ( N'__EFMigrationsHistory', N'__MigrationHistory', N'__FlywayHistory', N'sysdiagrams' ) THEN 1 " +
+                "WHEN o.object_name LIKE N'service_broker_%' THEN 1 WHEN o.object_name LIKE N'queue_messages_%' THEN 1 WHEN o.object_name LIKE N'MSmerge_%' " +
+                "THEN 1 WHEN o.object_name LIKE N'MSreplication_%' THEN 1 WHEN o.object_name LIKE N'FileTableUpdates$%' THEN 1 WHEN o.object_name LIKE N'graph_%' THEN 1 WHEN EXISTS " +
+                "( SELECT 1 FROM sys.tables AS t WHERE t.object_id = o.object_id AND ( t.is_tracked_by_cdc = 1 OR t.temporal_type > 0 OR t.is_filetable = 1 OR t.is_memory_optimized = 1 ) ) " +
+                "THEN 1 ELSE 0 END AS is_system_object FROM all_tables AS o JOIN sys.objects AS so ON so.object_id = o.object_id ) SELECT a.schema_name AS [schema], a.object_name AS [object], " +
+                "CASE WHEN LTRIM(RTRIM(ISNULL(@name_pattern, N''))) = N'' THEN a.object_name ELSE REPLACE( REPLACE(@name_pattern, N'{schema}', a.schema_name), N'{object}', a.object_name ) " +
+                "END AS entity_name, CASE WHEN EXISTS ( SELECT 1 FROM sys.columns AS c JOIN sys.types AS ty ON c.user_type_id = ty.user_type_id WHERE c.object_id = a.object_id AND ty.name IN " +
+                "( N'geography', N'geometry', N'hierarchyid', N'sql_variant', N'xml', N'rowversion', N'vector' ) ) THEN 1 ELSE 0 END AS contains_invalid_types FROM eligible_tables AS a WHERE " +
+                "a.is_system_object = 0 AND ( NOT EXISTS (SELECT 1 FROM exclude_patterns) OR NOT EXISTS ( SELECT 1 FROM exclude_patterns AS ep WHERE a.full_name LIKE ep.pattern " +
+                "COLLATE DATABASE_DEFAULT ESCAPE '\\' ) ) AND ( NOT EXISTS (SELECT 1 FROM include_patterns) OR EXISTS ( SELECT 1 FROM include_patterns AS ip WHERE a.full_name LIKE ip.pattern " +
+                "COLLATE DATABASE_DEFAULT ESCAPE '\\' ) ) AND ( @exclude_invalid_types = 0 OR NOT EXISTS ( SELECT 1 FROM sys.columns AS c JOIN sys.types AS ty ON c.user_type_id = ty.user_type_id " +
+                "WHERE c.object_id = a.object_id AND ty.name IN ( N'geography', N'geometry', N'hierarchyid', N'sql_variant', N'xml', N'rowversion', N'vector' ) ) ) ORDER BY a.schema_name, a.object_name;";
+
+            return query;
+        }
     }
 }

src/Core/Resolvers/MsSqlQueryExecutor.cs

@@ -82,7 +82,7 @@ public MsSqlQueryExecutor(
             _dataSourceToSessionContextUsage = new Dictionary<string, bool>();
             _accessTokensFromConfiguration = runtimeConfigProvider.ManagedIdentityAccessToken;
             _runtimeConfigProvider = runtimeConfigProvider;
-            ConfigureMsSqlQueryEecutor();
+            ConfigureMsSqlQueryExecutor();
         }
 
         /// <summary>
@@ -138,7 +138,7 @@ public override SqlConnection CreateConnection(string dataSourceName)
         /// <summary>
         /// Configure during construction or a hot-reload scenario.
         /// </summary>
-        private void ConfigureMsSqlQueryEecutor()
+        private void ConfigureMsSqlQueryExecutor()
         {
             IEnumerable<KeyValuePair<string, DataSource>> mssqldbs = _runtimeConfigProvider.GetConfig().GetDataSourceNamesToDataSourcesIterator().Where(x => x.Value.DatabaseType is DatabaseType.MSSQL || x.Value.DatabaseType is DatabaseType.DWSQL);
 

src/Core/Services/MetadataProviders/MsSqlMetadataProvider.cs

@@ -60,6 +60,7 @@ public override Type SqlToCLRType(string sqlType)
         /// <inheritdoc/>
         public override async Task PopulateTriggerMetadataForTable(string entityName, string schemaName, string tableName, SourceDefinition sourceDefinition)
         {
+
             string enumerateEnabledTriggers = SqlQueryBuilder.BuildFetchEnabledTriggersQuery();
             Dictionary<string, DbConnectionParam> parameters = new()
             {
@@ -290,5 +291,40 @@ private bool TryResolveDbType(string sqlDbTypeName, out DbType dbType)
                 return false;
             }
         }
+
+        /// <inheritdoc/>
+        protected override async Task GenerateAutoentitiesIntoEntities()
+        {
+            RuntimeConfig runtimeConfig = _runtimeConfigProvider.GetConfig();
+            if (runtimeConfig.Autoentities is not null)
+            {
+                foreach ((string name, Autoentity autoentity) in runtimeConfig.Autoentities.AutoEntities)
+                {
+                    JsonArray? resultArray = await QueryAutoentitiesAsync(autoentity);
+                    // TODO: Finish implementation of autoentities generation in task #3052
+                }
+            }
+        }
+
+        public async Task<JsonArray?> QueryAutoentitiesAsync(Autoentity autoentity)
+        {
+            string include = string.Join(",", autoentity.Patterns.Include);
+            string exclude = string.Join(",", autoentity.Patterns.Exclude);
+            string namePattern = autoentity.Patterns.Name;
+            string getAutoentitiesQuery = SqlQueryBuilder.BuildGetAutoentitiesQuery(include, exclude, namePattern);
+
+            _logger.LogInformation("Query for Autoentities is being executed with the following parameters.");
+            _logger.LogInformation($"Autoentities include pattern: {include}");
+            _logger.LogInformation($"Autoentities exclude pattern: {exclude}");
+            _logger.LogInformation($"Autoentities name pattern: {namePattern}");
+
+            JsonArray? resultArray = await QueryExecutor.ExecuteQueryAsync(
+                sqltext: getAutoentitiesQuery,
+                parameters: null!,
+                dataReaderHandler: QueryExecutor.GetJsonArrayAsync,
+                dataSourceName: _dataSourceName);
+
+            return resultArray;
+        }
     }
 }

src/Core/Services/MetadataProviders/SqlMetadataProvider.cs

@@ -307,6 +307,11 @@ public string GetEntityName(string graphQLType)
         public async Task InitializeAsync()
         {
             System.Diagnostics.Stopwatch timer = System.Diagnostics.Stopwatch.StartNew();
+            if (GetDatabaseType() == DatabaseType.MSSQL)
+            {
+                await GenerateAutoentitiesIntoEntities();
+            }
+
             GenerateDatabaseObjectForEntities();
             if (_isValidateOnly)
             {
@@ -686,6 +691,15 @@ private void GenerateDatabaseObjectForEntities()
             }
         }
 
+        /// <summary>
+        /// Creates entities for each table that is found, based on the autoentity configuration.
+        /// This method is only called for tables in MsSql.
+        /// </summary>
+        protected virtual Task GenerateAutoentitiesIntoEntities()
+        {
+            throw new NotSupportedException($"{GetType().Name} does not support Autoentities yet.");
+        }
+
         protected void PopulateDatabaseObjectForEntity(
             Entity entity,
             string entityName,

src/Service.Tests/UnitTests/SqlMetadataProviderUnitTests.cs

@@ -588,5 +588,68 @@ private static async Task SetupTestFixtureAndInferMetadata()
             await ResetDbStateAsync();
             await _sqlMetadataProvider.InitializeAsync();
         }
+
+        /// <summary>
+        /// Ensures that the query that returns the tables that will be generated
+        /// into entities from the autoentities configuration returns the expected result.
+        /// </summary>
+        [DataTestMethod, TestCategory(TestCategory.MSSQL)]
+        [DataRow(new string[] { "dbo.%book%" }, new string[] { }, "{schema}.{object}.books", new string[] { "book" }, "")]
+        [DataRow(new string[] { "dbo.%publish%" }, new string[] { }, "{schema}.{object}", new string[] { "publish" }, "")]
+        [DataRow(new string[] { "dbo.%book%" }, new string[] { "dbo.%books%" }, "{schema}_{object}_exclude_books", new string[] { "book" }, "books")]
+        [DataRow(new string[] { "dbo.%book%", "dbo.%publish%" }, new string[] { }, "{object}", new string[] { "book", "publish" }, "")]
+        [DataRow(new string[] { }, new string[] { "dbo.%book%" }, "{object}", new string[] { "" }, "book")]
+        public async Task CheckAutoentitiesQuery(string[] include, string[] exclude, string name, string[] includeObject, string excludeObject)
+        {
+            // Arrange
+            DatabaseEngine = TestCategory.MSSQL;
+            TestHelper.SetupDatabaseEnvironment(DatabaseEngine);
+            RuntimeConfig runtimeConfig = SqlTestHelper.SetupRuntimeConfig();
+            Autoentity autoentity = new(new AutoentityPatterns(include, exclude, name), null, null);
+            Dictionary<string, Autoentity> dictAutoentity = new()
+            {
+                { "autoentity", autoentity }
+            };
+            RuntimeConfig configWithAutoentity = runtimeConfig with
+            {
+                Autoentities = new RuntimeAutoentities(dictAutoentity)
+            };
+            RuntimeConfigProvider runtimeConfigProvider = TestHelper.GenerateInMemoryRuntimeConfigProvider(configWithAutoentity);
+            SetUpSQLMetadataProvider(runtimeConfigProvider);
+
+            await _sqlMetadataProvider.InitializeAsync();
+
+            // Act
+            MsSqlMetadataProvider metadataProvider = (MsSqlMetadataProvider)_sqlMetadataProvider;
+            JsonArray resultArray = await metadataProvider.QueryAutoentitiesAsync(autoentity);
+
+            // Assert
+            Assert.IsNotNull(resultArray);
+            foreach(JsonObject resultObject in resultArray)
+            {
+                bool includedObjectExists = false;
+                foreach (string included in includeObject)
+                {
+                    if (resultObject["object"].ToString().Contains(included))
+                    {
+                        includedObjectExists = true;
+                        Assert.AreNotEqual(name, resultObject["entity_name"].ToString(), "Name returned by query should not include {schema} or {object}.");
+                        if (include.Length > 0)
+                        {
+                            Assert.AreEqual(expected: "dbo", actual: resultObject["schema"].ToString(), "Query does not return expected schema.");
+                        }
+
+                        if (exclude.Length > 0)
+                        {
+                            Assert.IsTrue(!resultObject["object"].ToString().Contains(excludeObject), "Query returns pattern that should be excluded.");
+                        }
+                    }
+                }
+
+                Assert.IsTrue(includedObjectExists, "Query does not return expected object.");
+            }
+
+            TestHelper.UnsetAllDABEnvironmentVariables();
+        }
     }
 }