From 54fb1b115fa85e9c3fa4e985f65c9263b3baf159 Mon Sep 17 00:00:00 2001 From: Joymax Andrade do Nascimento Date: Fri, 4 Sep 2026 17:29:33 -0300 Subject: [PATCH 01/19] Honor field permissions when reading table schema --- .../MetadataProviders/SqlMetadataProvider.cs | 349 +++++++++++++++++- 1 file changed, 346 insertions(+), 3 deletions(-) diff --git a/src/Core/Services/MetadataProviders/SqlMetadataProvider.cs b/src/Core/Services/MetadataProviders/SqlMetadataProvider.cs index 9517c41781..6b31d7c700 100644 --- a/src/Core/Services/MetadataProviders/SqlMetadataProvider.cs +++ b/src/Core/Services/MetadataProviders/SqlMetadataProvider.cs @@ -69,6 +69,11 @@ public abstract class SqlMetadataProvider : protected const int NUMBER_OF_RESTRICTIONS = 4; + /// + /// Wildcard used in the permissions "fields" section to denote every field. + /// + private const string FIELD_WILDCARD = "*"; + protected string ConnectionString { get; init; } protected IQueryBuilder SqlQueryBuilder { get; init; } @@ -1528,10 +1533,26 @@ private async Task PopulateSourceDefinitionAsync( using DataTableReader reader = new(dataTable); DataTable schemaTable = reader.GetSchemaTable(); RuntimeConfig runtimeConfig = _runtimeConfigProvider.GetConfig(); + + // Columns the entity's permissions allow to be read. The schema DataTable is cached + // per schema.table and may therefore carry columns permitted only for a sibling + // entity, so the restriction is re-applied per entity here. + PermittedColumns? permittedColumns = entity is null + ? null + : ResolvePermittedColumnsForEntity(entity); + foreach (DataRow columnInfoFromAdapter in schemaTable.Rows) { string columnName = columnInfoFromAdapter["ColumnName"].ToString()!; + if (permittedColumns is not null + && !permittedColumns.IsUnrestricted + && !permittedColumns.IsColumnPermitted(columnName) + && !sourceDefinition.PrimaryKey.Contains(columnName)) + { + continue; + } + if (runtimeConfig.IsGraphQLEnabled && entity is not null && IsGraphQLReservedName(entity, columnName, graphQLEnabledGlobally: runtimeConfig.IsGraphQLEnabled)) @@ -1685,7 +1706,7 @@ private async Task GetTableWithSchemaFromDataSetAsync( { try { - dataTable = await FillSchemaForTableAsync(schemaName, tableName); + dataTable = await FillSchemaForTableAsync(schemaName, tableName, entityName); } catch (Exception ex) when (ex is not DataApiBuilderException) { @@ -1753,10 +1774,16 @@ private async Task ValidateDatabaseConnection() /// /// Using a data adapter, obtains the schema of the given table name /// and adds the corresponding DataTable to the entities data set. + /// When the entities backed by this database object restrict the readable + /// fields through permissions ("fields.include"/"fields.exclude"), the projection + /// is narrowed to those columns instead of "SELECT *". This avoids the provider + /// having to materialize CLR types it cannot handle (e.g. geometry/geography/hierarchyid), + /// which otherwise fails during schema discovery even though the column is not exposed. /// private async Task FillSchemaForTableAsync( string schemaName, - string tableName) + string tableName, + string? entityName = null) { using ConnectionT conn = new(); // If connection string is set to empty string @@ -1802,14 +1829,330 @@ private async Task FillSchemaForTableAsync( }; string tableNameWithSchemaPrefix = GetTableNameWithSchemaPrefix(schemaName, tableName); + + // Resolve the columns the configuration actually allows to be read for this + // database object. When nothing is restricted, the original SELECT * is preserved. + PermittedColumns permittedColumns = ResolvePermittedColumnsForDatabaseObject( + schemaName: schemaName, + tableName: tableName, + entityName: entityName); + + string projection = await BuildSchemaProjectionAsync(schemaName, tableName, permittedColumns); + selectCommand.CommandText - = $"SELECT * FROM {tableNameWithSchemaPrefix}"; + = $"SELECT {projection} FROM {tableNameWithSchemaPrefix}"; adapterForTable.SelectCommand = selectCommand; DataTable[] dataTable = adapterForTable.FillSchema(EntitiesDataSet, SchemaType.Source, tableNameWithSchemaPrefix); return dataTable[0]; } + /// + /// Describes which backing (database) columns of a database object the runtime + /// configuration allows to be read. + /// + /// + /// True when at least one permission reads every field ("fields" absent, "include" absent, + /// or "include": ["*"]). + /// + /// Backing columns explicitly listed in an "include" section. + /// + /// Backing columns excluded from every wildcard permission and never explicitly included. + /// + private sealed record PermittedColumns(bool AllColumns, HashSet Included, HashSet Excluded) + { + /// + /// No column restriction could be derived from the configuration. + /// + public bool IsUnrestricted => AllColumns && Excluded.Count == 0; + + /// + /// Whether the given backing column is readable per the configuration. + /// + public bool IsColumnPermitted(string columnName) + { + if (Included.Contains(columnName)) + { + return true; + } + + if (Excluded.Contains(columnName)) + { + return false; + } + + return AllColumns; + } + } + + /// + /// Resolves the readable columns for a database object. + /// Because the schema DataTable is cached per schema.table, the result combines the + /// permissions of every entity in this data source backed by that same object: a column + /// has to be read if any of those entities can read it. + /// + /// Schema of the database object. + /// Name of the database object. + /// Entity that triggered the schema discovery, when known. + private PermittedColumns ResolvePermittedColumnsForDatabaseObject( + string schemaName, + string tableName, + string? entityName) + { + HashSet included = new(StringComparer.Ordinal); + HashSet? excluded = null; + bool allColumns = false; + bool matchedAnyEntity = false; + + foreach ((string candidateEntityName, Entity candidateEntity) in Entities) + { + // Only consider entities backed by the same database object. + if (EntityToDatabaseObject.TryGetValue(candidateEntityName, out DatabaseObject? databaseObject)) + { + if (!string.Equals(databaseObject.SchemaName, schemaName, StringComparison.OrdinalIgnoreCase) + || !string.Equals(databaseObject.Name, tableName, StringComparison.OrdinalIgnoreCase)) + { + continue; + } + } + else if (!string.Equals(candidateEntityName, entityName, StringComparison.Ordinal)) + { + // Database object not inferred yet: only the entity that triggered the read applies. + continue; + } + + matchedAnyEntity = true; + PermittedColumns entityPermittedColumns = ResolvePermittedColumnsForEntity(candidateEntity); + + included.UnionWith(entityPermittedColumns.Included); + + if (entityPermittedColumns.AllColumns) + { + allColumns = true; + + // A column is only droppable when every wildcard permission excludes it. + if (excluded is null) + { + excluded = new(entityPermittedColumns.Excluded, StringComparer.Ordinal); + } + else + { + excluded.IntersectWith(entityPermittedColumns.Excluded); + } + } + } + + if (!matchedAnyEntity) + { + return new(AllColumns: true, Included: new(StringComparer.Ordinal), Excluded: new(StringComparer.Ordinal)); + } + + excluded ??= new(StringComparer.Ordinal); + excluded.ExceptWith(included); + + return new(allColumns, included, excluded); + } + + /// + /// Resolves the readable columns of a single entity from the "fields.include" / + /// "fields.exclude" sections of its permissions. Exposed names (mappings and field + /// aliases) are translated back to their database column names, and configured primary + /// key fields are always kept, since the runtime cannot operate on the entity without them. + /// + private static PermittedColumns ResolvePermittedColumnsForEntity(Entity entity) + { + HashSet included = new(StringComparer.Ordinal); + HashSet? excluded = null; + bool allColumns = false; + + // Exposed name -> backing column name. + Dictionary exposedToBackingName = new(StringComparer.Ordinal); + + if (entity.Mappings is not null) + { + foreach ((string backingName, string exposedName) in entity.Mappings) + { + if (!string.IsNullOrWhiteSpace(exposedName)) + { + exposedToBackingName[exposedName] = backingName; + } + } + } + + if (entity.Fields is not null) + { + foreach (FieldMetadata field in entity.Fields) + { + if (!string.IsNullOrWhiteSpace(field.Alias)) + { + exposedToBackingName[field.Alias!] = field.Name; + } + } + } + + if (entity.Permissions is null || entity.Permissions.Length == 0) + { + // Nothing configured: the whole object is read, as before. + return new(AllColumns: true, included, new HashSet(StringComparer.Ordinal)); + } + + foreach (EntityPermission permission in entity.Permissions) + { + if (permission.Actions is null) + { + allColumns = true; + excluded = new(StringComparer.Ordinal); + continue; + } + + foreach (EntityAction action in permission.Actions) + { + EntityActionFields? fields = action.Fields; + + HashSet actionExcluded = new(StringComparer.Ordinal); + if (fields?.Exclude is not null) + { + if (fields.Exclude.Contains(FIELD_WILDCARD)) + { + // This permission reads no field at all, so it contributes no column. + continue; + } + + foreach (string field in fields.Exclude) + { + actionExcluded.Add(ResolveBackingName(field, exposedToBackingName)); + } + } + + // No "fields" section, no "include" section, or an explicit wildcard, + // means every column not listed in "exclude" is readable. + bool includesEveryField = fields is null + || fields.Include is null + || fields.Include.Contains(FIELD_WILDCARD); + + if (includesEveryField) + { + allColumns = true; + + if (excluded is null) + { + excluded = actionExcluded; + } + else + { + excluded.IntersectWith(actionExcluded); + } + + continue; + } + + foreach (string field in fields!.Include!) + { + string backingName = ResolveBackingName(field, exposedToBackingName); + if (!actionExcluded.Contains(backingName)) + { + included.Add(backingName); + } + } + } + } + + // Primary keys are structural: never drop them from the projection. + if (entity.Source is not null && entity.Source.KeyFields is not null) + { + foreach (string keyField in entity.Source.KeyFields) + { + included.Add(ResolveBackingName(keyField, exposedToBackingName)); + } + } + + if (entity.Fields is not null) + { + foreach (FieldMetadata field in entity.Fields.Where(f => f.PrimaryKey)) + { + included.Add(ResolveBackingName(field.Name, exposedToBackingName)); + } + } + + excluded ??= new(StringComparer.Ordinal); + excluded.ExceptWith(included); + + return new(allColumns, included, excluded); + } + + /// + /// Translates a configured (exposed) field name into its backing column name. + /// + private static string ResolveBackingName(string fieldName, Dictionary exposedToBackingName) + { + return exposedToBackingName.TryGetValue(fieldName, out string? backingName) ? backingName : fieldName; + } + + /// + /// Builds the projection used to read the schema of a database object, narrowed to the + /// columns the configuration allows to be read. Returns "*" when no restriction applies + /// or when the column list cannot be determined. + /// + private async Task BuildSchemaProjectionAsync( + string schemaName, + string tableName, + PermittedColumns permittedColumns) + { + if (permittedColumns.IsUnrestricted) + { + return "*"; + } + + List columnsToRead; + + if (permittedColumns.AllColumns) + { + // "include": ["*"] with an "exclude" list: enumerate the columns from the catalog + // (metadata only, so unsupported CLR types are never materialized) and drop the + // excluded ones. + List allColumnNames = new(); + try + { + DataTable columnsInTable = await GetColumnsAsync(schemaName, tableName); + + foreach (DataRow columnInfo in columnsInTable.Rows) + { + if (columnInfo["COLUMN_NAME"] is string columnName) + { + allColumnNames.Add(columnName); + } + } + } + catch (Exception ex) + { + _logger.LogDebug( + "Unable to enumerate the columns of {schemaName}.{tableName} to honor the configured field exclusions: {message}", + schemaName, + tableName, + ex.Message); + return "*"; + } + + if (allColumnNames.Count == 0) + { + return "*"; + } + + columnsToRead = allColumnNames.Where(permittedColumns.IsColumnPermitted).ToList(); + } + else + { + columnsToRead = permittedColumns.Included.ToList(); + } + + if (columnsToRead.Count == 0) + { + return "*"; + } + + return string.Join(", ", columnsToRead.Select(column => SqlQueryBuilder.QuoteIdentifier(column))); + } + /// /// Gets the correctly formatted table name with schema as prefix, if one exists. /// A schema prefix is simply the correctly formatted and prefixed schema name that From c330065e9b0d3de3d5599f24bccdc96425d05007 Mon Sep 17 00:00:00 2001 From: Joymax Andrade do Nascimento Date: Fri, 4 Sep 2026 21:13:00 -0300 Subject: [PATCH 02/19] Add MSSQL fixture and test for column excluded by field permissions --- config-generators/mssql-commands.txt | 1 + src/Service.Tests/DatabaseSchema-MsSql.sql | 14 ++++++++ ...tReadingRuntimeConfigForMsSql.verified.txt | 32 ++++++++++++++++++ .../UnitTests/SqlMetadataProviderUnitTests.cs | 32 ++++++++++++++++++ src/Service.Tests/dab-config.MsSql.json | 33 +++++++++++++++++++ 5 files changed, 112 insertions(+) diff --git a/config-generators/mssql-commands.txt b/config-generators/mssql-commands.txt index 277b9878f0..0b1fe8ca1d 100644 --- a/config-generators/mssql-commands.txt +++ b/config-generators/mssql-commands.txt @@ -22,6 +22,7 @@ add VectorType --config "dab-config.MsSql.json" --source vector_type_table --res update VectorType --config "dab-config.MsSql.json" --permissions "authenticated:create,read,delete,update" update VectorOwner --config "dab-config.MsSql.json" --relationship vectors --target.entity VectorType --cardinality many --relationship.fields "id:owner_id" update VectorType --config "dab-config.MsSql.json" --relationship owner --target.entity VectorOwner --cardinality one --relationship.fields "owner_id:id" +add GeometryType --config "dab-config.MsSql.json" --source geometry_type_table --rest true --graphql true --permissions "anonymous:read" --fields.include "id,name" add Profile --config "dab-config.MsSql.json" --source profiles --rest true --graphql true --permissions "anonymous:create,read,delete,update" update Profile --config "dab-config.MsSql.json" --permissions "authenticated:create,read,delete,update" add stocks_price --config "dab-config.MsSql.json" --source stocks_price --permissions "authenticated:create,read,update,delete" diff --git a/src/Service.Tests/DatabaseSchema-MsSql.sql b/src/Service.Tests/DatabaseSchema-MsSql.sql index 587edb29c3..ce63144076 100644 --- a/src/Service.Tests/DatabaseSchema-MsSql.sql +++ b/src/Service.Tests/DatabaseSchema-MsSql.sql @@ -44,6 +44,7 @@ DROP TABLE IF EXISTS brokers; DROP TABLE IF EXISTS type_table; DROP TABLE IF EXISTS vector_type_table; DROP TABLE IF EXISTS vector_owners; +DROP TABLE IF EXISTS geometry_type_table; DROP TABLE IF EXISTS profiles; DROP TABLE IF EXISTS trees; DROP TABLE IF EXISTS fungi; @@ -252,6 +253,12 @@ CREATE TABLE vector_type_table( CONSTRAINT FK_vector_type_table_owner FOREIGN KEY (owner_id) REFERENCES vector_owners(id) ON DELETE CASCADE ); +CREATE TABLE geometry_type_table( + id int IDENTITY(5001, 1) PRIMARY KEY, + name varchar(100) NOT NULL, + geom geometry NULL +); + CREATE TABLE profiles( id int IDENTITY(5001, 1) PRIMARY KEY, metadata json NULL @@ -656,6 +663,13 @@ VALUES (7, CAST('[' + ( ) + ']' AS vector(1998))); SET IDENTITY_INSERT vector_type_table OFF +SET IDENTITY_INSERT geometry_type_table ON +INSERT INTO geometry_type_table(id, name, geom) +VALUES + (1, 'point', geometry::STGeomFromText('POINT(1 2)', 0)), + (2, 'null geometry', NULL); +SET IDENTITY_INSERT geometry_type_table OFF + SET IDENTITY_INSERT profiles ON INSERT INTO profiles(id, metadata) VALUES diff --git a/src/Service.Tests/Snapshots/ConfigurationTests.TestReadingRuntimeConfigForMsSql.verified.txt b/src/Service.Tests/Snapshots/ConfigurationTests.TestReadingRuntimeConfigForMsSql.verified.txt index 3c3b8224e4..57e7548405 100644 --- a/src/Service.Tests/Snapshots/ConfigurationTests.TestReadingRuntimeConfigForMsSql.verified.txt +++ b/src/Service.Tests/Snapshots/ConfigurationTests.TestReadingRuntimeConfigForMsSql.verified.txt @@ -1910,6 +1910,38 @@ } } }, + { + GeometryType: { + Source: { + Object: geometry_type_table, + Type: Table + }, + GraphQL: { + Singular: GeometryType, + Plural: GeometryTypes, + Enabled: true + }, + Rest: { + Enabled: true + }, + Permissions: [ + { + Role: anonymous, + Actions: [ + { + Action: Read, + Fields: { + Include: [ + id, + name + ] + } + } + ] + } + ] + } + }, { Profile: { Source: { diff --git a/src/Service.Tests/UnitTests/SqlMetadataProviderUnitTests.cs b/src/Service.Tests/UnitTests/SqlMetadataProviderUnitTests.cs index dd6ad7d27e..50ed2026f0 100644 --- a/src/Service.Tests/UnitTests/SqlMetadataProviderUnitTests.cs +++ b/src/Service.Tests/UnitTests/SqlMetadataProviderUnitTests.cs @@ -396,6 +396,38 @@ public async Task ValidateInferredRelationshipInfoForMsSql() ValidateInferredRelationshipInfoForTables(); } + /// + /// Test to validate that a table holding a column whose CLR type the data provider cannot + /// resolve - here a geometry column - is still usable when the entity permissions enumerate + /// the readable fields and that column is not among them. + /// Metadata inference must succeed and the unreadable column must be absent from the + /// inferred source definition, so it never reaches the OData or GraphQL type maps. + /// + [TestMethod, TestCategory(TestCategory.MSSQL)] + public async Task ValidateColumnExcludedByFieldPermissionsIsNotInferred() + { + DatabaseEngine = TestCategory.MSSQL; + await SetupTestFixtureAndInferMetadata(); + + Assert.IsTrue( + _sqlMetadataProvider.GetEntityNamesAndDbObjects().TryGetValue("GeometryType", out DatabaseObject databaseObject), + message: "Metadata inference failed for the entity backed by a table with a geometry column."); + + SourceDefinition sourceDefinition = databaseObject.SourceDefinition; + + Assert.IsTrue( + sourceDefinition.Columns.ContainsKey("id"), + message: "The primary key column is expected in the source definition."); + Assert.IsTrue( + sourceDefinition.Columns.ContainsKey("name"), + message: "A column listed in fields.include is expected in the source definition."); + Assert.IsFalse( + sourceDefinition.Columns.ContainsKey("geom"), + message: "A column absent from fields.include is not expected in the source definition."); + + TestHelper.UnsetAllDABEnvironmentVariables(); + } + /// /// Test to validate successful inference of relationship data based on data provided in the config and the metadata /// collected from the MySql database. diff --git a/src/Service.Tests/dab-config.MsSql.json b/src/Service.Tests/dab-config.MsSql.json index 670f390d4c..fa80aa2e29 100644 --- a/src/Service.Tests/dab-config.MsSql.json +++ b/src/Service.Tests/dab-config.MsSql.json @@ -1990,6 +1990,39 @@ } } }, + "GeometryType": { + "source": { + "object": "geometry_type_table", + "type": "table" + }, + "graphql": { + "enabled": true, + "type": { + "singular": "GeometryType", + "plural": "GeometryTypes" + } + }, + "rest": { + "enabled": true + }, + "permissions": [ + { + "role": "anonymous", + "actions": [ + { + "action": "read", + "fields": { + "exclude": [], + "include": [ + "id", + "name" + ] + } + } + ] + } + ] + }, "Profile": { "source": { "object": "profiles", From 0d1330376e652e1b8589879e7137628ae717e3e1 Mon Sep 17 00:00:00 2001 From: Joymax Andrade do Nascimento Date: Fri, 4 Sep 2026 21:37:01 -0300 Subject: [PATCH 03/19] Match column names case-insensitively when resolving field permissions --- .../MetadataProviders/SqlMetadataProvider.cs | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/src/Core/Services/MetadataProviders/SqlMetadataProvider.cs b/src/Core/Services/MetadataProviders/SqlMetadataProvider.cs index 6b31d7c700..1b9c10ebac 100644 --- a/src/Core/Services/MetadataProviders/SqlMetadataProvider.cs +++ b/src/Core/Services/MetadataProviders/SqlMetadataProvider.cs @@ -1548,7 +1548,7 @@ private async Task PopulateSourceDefinitionAsync( if (permittedColumns is not null && !permittedColumns.IsUnrestricted && !permittedColumns.IsColumnPermitted(columnName) - && !sourceDefinition.PrimaryKey.Contains(columnName)) + && !sourceDefinition.PrimaryKey.Contains(columnName, StringComparer.OrdinalIgnoreCase)) { continue; } @@ -1850,6 +1850,9 @@ private async Task FillSchemaForTableAsync( /// /// Describes which backing (database) columns of a database object the runtime /// configuration allows to be read. + /// Column names are matched case-insensitively, consistent with the comparer used by + /// SourceDefinition.Columns and with the field name lookups in this class, so a + /// configuration whose casing differs from the database schema still resolves. /// /// /// True when at least one permission reads every field ("fields" absent, "include" absent, @@ -1899,7 +1902,7 @@ private PermittedColumns ResolvePermittedColumnsForDatabaseObject( string tableName, string? entityName) { - HashSet included = new(StringComparer.Ordinal); + HashSet included = new(StringComparer.OrdinalIgnoreCase); HashSet? excluded = null; bool allColumns = false; bool matchedAnyEntity = false; @@ -1933,7 +1936,7 @@ private PermittedColumns ResolvePermittedColumnsForDatabaseObject( // A column is only droppable when every wildcard permission excludes it. if (excluded is null) { - excluded = new(entityPermittedColumns.Excluded, StringComparer.Ordinal); + excluded = new(entityPermittedColumns.Excluded, StringComparer.OrdinalIgnoreCase); } else { @@ -1944,10 +1947,10 @@ private PermittedColumns ResolvePermittedColumnsForDatabaseObject( if (!matchedAnyEntity) { - return new(AllColumns: true, Included: new(StringComparer.Ordinal), Excluded: new(StringComparer.Ordinal)); + return new(AllColumns: true, Included: new(StringComparer.OrdinalIgnoreCase), Excluded: new(StringComparer.OrdinalIgnoreCase)); } - excluded ??= new(StringComparer.Ordinal); + excluded ??= new(StringComparer.OrdinalIgnoreCase); excluded.ExceptWith(included); return new(allColumns, included, excluded); @@ -1961,12 +1964,12 @@ private PermittedColumns ResolvePermittedColumnsForDatabaseObject( /// private static PermittedColumns ResolvePermittedColumnsForEntity(Entity entity) { - HashSet included = new(StringComparer.Ordinal); + HashSet included = new(StringComparer.OrdinalIgnoreCase); HashSet? excluded = null; bool allColumns = false; // Exposed name -> backing column name. - Dictionary exposedToBackingName = new(StringComparer.Ordinal); + Dictionary exposedToBackingName = new(StringComparer.OrdinalIgnoreCase); if (entity.Mappings is not null) { @@ -1993,7 +1996,7 @@ private static PermittedColumns ResolvePermittedColumnsForEntity(Entity entity) if (entity.Permissions is null || entity.Permissions.Length == 0) { // Nothing configured: the whole object is read, as before. - return new(AllColumns: true, included, new HashSet(StringComparer.Ordinal)); + return new(AllColumns: true, included, new HashSet(StringComparer.OrdinalIgnoreCase)); } foreach (EntityPermission permission in entity.Permissions) @@ -2001,7 +2004,7 @@ private static PermittedColumns ResolvePermittedColumnsForEntity(Entity entity) if (permission.Actions is null) { allColumns = true; - excluded = new(StringComparer.Ordinal); + excluded = new(StringComparer.OrdinalIgnoreCase); continue; } @@ -2009,7 +2012,7 @@ private static PermittedColumns ResolvePermittedColumnsForEntity(Entity entity) { EntityActionFields? fields = action.Fields; - HashSet actionExcluded = new(StringComparer.Ordinal); + HashSet actionExcluded = new(StringComparer.OrdinalIgnoreCase); if (fields?.Exclude is not null) { if (fields.Exclude.Contains(FIELD_WILDCARD)) @@ -2074,7 +2077,7 @@ private static PermittedColumns ResolvePermittedColumnsForEntity(Entity entity) } } - excluded ??= new(StringComparer.Ordinal); + excluded ??= new(StringComparer.OrdinalIgnoreCase); excluded.ExceptWith(included); return new(allColumns, included, excluded); From 9b7d458154acbef6a6869eea7a2d2cba5eb3e058 Mon Sep 17 00:00:00 2001 From: Joymax Andrade do Nascimento Date: Fri, 4 Sep 2026 21:59:27 -0300 Subject: [PATCH 04/19] Only narrow the schema projection when the primary key is configured --- .../MetadataProviders/SqlMetadataProvider.cs | 53 ++++++++++++------- 1 file changed, 33 insertions(+), 20 deletions(-) diff --git a/src/Core/Services/MetadataProviders/SqlMetadataProvider.cs b/src/Core/Services/MetadataProviders/SqlMetadataProvider.cs index 1b9c10ebac..d9b26da03d 100644 --- a/src/Core/Services/MetadataProviders/SqlMetadataProvider.cs +++ b/src/Core/Services/MetadataProviders/SqlMetadataProvider.cs @@ -1959,9 +1959,17 @@ private PermittedColumns ResolvePermittedColumnsForDatabaseObject( /// /// Resolves the readable columns of a single entity from the "fields.include" / /// "fields.exclude" sections of its permissions. Exposed names (mappings and field - /// aliases) are translated back to their database column names, and configured primary - /// key fields are always kept, since the runtime cannot operate on the entity without them. + /// aliases) are translated back to their database column names, and the configured + /// primary key is always kept, since the runtime cannot operate on the entity without it. /// + /// + /// The projection is only narrowed for entities whose primary key is known from the + /// configuration ("fields[].primary-key" or "source.key-fields"). When the primary key is + /// instead inferred from the schema read itself - see the fallback to + /// DataTable.PrimaryKey in PopulateObjectDefinitionForEntity - narrowing + /// could drop the key column and leave the entity with no primary key, so every column is + /// read as before. + /// private static PermittedColumns ResolvePermittedColumnsForEntity(Entity entity) { HashSet included = new(StringComparer.OrdinalIgnoreCase); @@ -1993,9 +2001,28 @@ private static PermittedColumns ResolvePermittedColumnsForEntity(Entity entity) } } - if (entity.Permissions is null || entity.Permissions.Length == 0) + HashSet configuredPrimaryKey = new(StringComparer.OrdinalIgnoreCase); + + if (entity.Fields is not null) + { + foreach (FieldMetadata field in entity.Fields.Where(f => f.PrimaryKey)) + { + configuredPrimaryKey.Add(ResolveBackingName(field.Name, exposedToBackingName)); + } + } + + if (configuredPrimaryKey.Count == 0 && entity.Source is not null && entity.Source.KeyFields is not null) + { + foreach (string keyField in entity.Source.KeyFields) + { + configuredPrimaryKey.Add(ResolveBackingName(keyField, exposedToBackingName)); + } + } + + // Without a configured primary key, the key itself is inferred from this schema read, + // so a narrowed projection could drop it and leave the entity unusable. Read it all. + if (entity.Permissions is null || entity.Permissions.Length == 0 || configuredPrimaryKey.Count == 0) { - // Nothing configured: the whole object is read, as before. return new(AllColumns: true, included, new HashSet(StringComparer.OrdinalIgnoreCase)); } @@ -2060,22 +2087,8 @@ private static PermittedColumns ResolvePermittedColumnsForEntity(Entity entity) } } - // Primary keys are structural: never drop them from the projection. - if (entity.Source is not null && entity.Source.KeyFields is not null) - { - foreach (string keyField in entity.Source.KeyFields) - { - included.Add(ResolveBackingName(keyField, exposedToBackingName)); - } - } - - if (entity.Fields is not null) - { - foreach (FieldMetadata field in entity.Fields.Where(f => f.PrimaryKey)) - { - included.Add(ResolveBackingName(field.Name, exposedToBackingName)); - } - } + // The primary key is structural: never drop it from the projection. + included.UnionWith(configuredPrimaryKey); excluded ??= new(StringComparer.OrdinalIgnoreCase); excluded.ExceptWith(included); From 7a82fa7865b8b1e120fd7f858f916fce4efc9714 Mon Sep 17 00:00:00 2001 From: Joymax Andrade do Nascimento Date: Fri, 4 Sep 2026 22:22:11 -0300 Subject: [PATCH 05/19] Skip columns whose data type the provider cannot map, instead of honoring field permissions --- config-generators/mssql-commands.txt | 2 +- .../MsSqlMetadataProvider.cs | 18 + .../MetadataProviders/SqlMetadataProvider.cs | 377 +++--------------- ...tReadingRuntimeConfigForMsSql.verified.txt | 8 +- .../UnitTests/SqlMetadataProviderUnitTests.cs | 17 +- src/Service.Tests/dab-config.MsSql.json | 9 +- 6 files changed, 82 insertions(+), 349 deletions(-) diff --git a/config-generators/mssql-commands.txt b/config-generators/mssql-commands.txt index 0b1fe8ca1d..53674ef59c 100644 --- a/config-generators/mssql-commands.txt +++ b/config-generators/mssql-commands.txt @@ -22,7 +22,7 @@ add VectorType --config "dab-config.MsSql.json" --source vector_type_table --res update VectorType --config "dab-config.MsSql.json" --permissions "authenticated:create,read,delete,update" update VectorOwner --config "dab-config.MsSql.json" --relationship vectors --target.entity VectorType --cardinality many --relationship.fields "id:owner_id" update VectorType --config "dab-config.MsSql.json" --relationship owner --target.entity VectorOwner --cardinality one --relationship.fields "owner_id:id" -add GeometryType --config "dab-config.MsSql.json" --source geometry_type_table --rest true --graphql true --permissions "anonymous:read" --fields.include "id,name" +add GeometryType --config "dab-config.MsSql.json" --source geometry_type_table --rest true --graphql true --permissions "anonymous:read" add Profile --config "dab-config.MsSql.json" --source profiles --rest true --graphql true --permissions "anonymous:create,read,delete,update" update Profile --config "dab-config.MsSql.json" --permissions "authenticated:create,read,delete,update" add stocks_price --config "dab-config.MsSql.json" --source stocks_price --permissions "authenticated:create,read,update,delete" diff --git a/src/Core/Services/MetadataProviders/MsSqlMetadataProvider.cs b/src/Core/Services/MetadataProviders/MsSqlMetadataProvider.cs index 20de74a96d..9176b60566 100644 --- a/src/Core/Services/MetadataProviders/MsSqlMetadataProvider.cs +++ b/src/Core/Services/MetadataProviders/MsSqlMetadataProvider.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +using System.Collections.Immutable; using System.Data; using System.Data.Common; using System.Net; @@ -44,6 +45,23 @@ public MsSqlMetadataProvider( _runtimeConfigProvider = runtimeConfigProvider; } + /// + /// SQL Server CLR user-defined types. Microsoft.Data.SqlClient resolves their CLR type + /// through the Microsoft.SqlServer.Types assembly, which Data API builder does not + /// reference, so the reader reports no type for the column and the data adapter fails. + /// Deliberately limited to the types that cannot be read at all: timestamp, xml and vector + /// columns do resolve to a CLR type and are left untouched. + /// + private static readonly ImmutableHashSet _unsupportedColumnDataTypes = + ImmutableHashSet.Create( + StringComparer.OrdinalIgnoreCase, + "geometry", + "geography", + "hierarchyid"); + + /// + protected override ImmutableHashSet UnsupportedColumnDataTypes => _unsupportedColumnDataTypes; + public override string GetDefaultSchemaName() { return "dbo"; diff --git a/src/Core/Services/MetadataProviders/SqlMetadataProvider.cs b/src/Core/Services/MetadataProviders/SqlMetadataProvider.cs index d9b26da03d..d801290044 100644 --- a/src/Core/Services/MetadataProviders/SqlMetadataProvider.cs +++ b/src/Core/Services/MetadataProviders/SqlMetadataProvider.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +using System.Collections.Immutable; using System.Collections.ObjectModel; using System.Data; using System.Data.Common; @@ -70,9 +71,15 @@ public abstract class SqlMetadataProvider : protected const int NUMBER_OF_RESTRICTIONS = 4; /// - /// Wildcard used in the permissions "fields" section to denote every field. + /// Column data types, as reported by the "Columns" schema collection, that the data + /// provider cannot map to a CLR type. Reading such a column makes + /// fail with + /// "DataReader.GetFieldType(N) returned null", which takes down the whole database object + /// even when the column itself is never exposed. They are therefore left out of the + /// projection used for schema discovery. + /// Empty by default: a provider only lists a type here when it genuinely cannot resolve it. /// - private const string FIELD_WILDCARD = "*"; + protected virtual ImmutableHashSet UnsupportedColumnDataTypes => ImmutableHashSet.Empty; protected string ConnectionString { get; init; } @@ -1534,25 +1541,10 @@ private async Task PopulateSourceDefinitionAsync( DataTable schemaTable = reader.GetSchemaTable(); RuntimeConfig runtimeConfig = _runtimeConfigProvider.GetConfig(); - // Columns the entity's permissions allow to be read. The schema DataTable is cached - // per schema.table and may therefore carry columns permitted only for a sibling - // entity, so the restriction is re-applied per entity here. - PermittedColumns? permittedColumns = entity is null - ? null - : ResolvePermittedColumnsForEntity(entity); - foreach (DataRow columnInfoFromAdapter in schemaTable.Rows) { string columnName = columnInfoFromAdapter["ColumnName"].ToString()!; - if (permittedColumns is not null - && !permittedColumns.IsUnrestricted - && !permittedColumns.IsColumnPermitted(columnName) - && !sourceDefinition.PrimaryKey.Contains(columnName, StringComparer.OrdinalIgnoreCase)) - { - continue; - } - if (runtimeConfig.IsGraphQLEnabled && entity is not null && IsGraphQLReservedName(entity, columnName, graphQLEnabledGlobally: runtimeConfig.IsGraphQLEnabled)) @@ -1706,7 +1698,7 @@ private async Task GetTableWithSchemaFromDataSetAsync( { try { - dataTable = await FillSchemaForTableAsync(schemaName, tableName, entityName); + dataTable = await FillSchemaForTableAsync(schemaName, tableName); } catch (Exception ex) when (ex is not DataApiBuilderException) { @@ -1774,16 +1766,13 @@ private async Task ValidateDatabaseConnection() /// /// Using a data adapter, obtains the schema of the given table name /// and adds the corresponding DataTable to the entities data set. - /// When the entities backed by this database object restrict the readable - /// fields through permissions ("fields.include"/"fields.exclude"), the projection - /// is narrowed to those columns instead of "SELECT *". This avoids the provider - /// having to materialize CLR types it cannot handle (e.g. geometry/geography/hierarchyid), - /// which otherwise fails during schema discovery even though the column is not exposed. + /// Columns whose data type the data provider cannot map to a CLR type are left out of the + /// projection, because the data adapter refuses to build a schema mapping for them and the + /// whole object would otherwise be unreachable. See . /// private async Task FillSchemaForTableAsync( string schemaName, - string tableName, - string? entityName = null) + string tableName) { using ConnectionT conn = new(); // If connection string is set to empty string @@ -1830,14 +1819,7 @@ private async Task FillSchemaForTableAsync( string tableNameWithSchemaPrefix = GetTableNameWithSchemaPrefix(schemaName, tableName); - // Resolve the columns the configuration actually allows to be read for this - // database object. When nothing is restricted, the original SELECT * is preserved. - PermittedColumns permittedColumns = ResolvePermittedColumnsForDatabaseObject( - schemaName: schemaName, - tableName: tableName, - entityName: entityName); - - string projection = await BuildSchemaProjectionAsync(schemaName, tableName, permittedColumns); + string projection = await BuildSchemaProjectionAsync(schemaName, tableName); selectCommand.CommandText = $"SELECT {projection} FROM {tableNameWithSchemaPrefix}"; @@ -1848,325 +1830,70 @@ private async Task FillSchemaForTableAsync( } /// - /// Describes which backing (database) columns of a database object the runtime - /// configuration allows to be read. - /// Column names are matched case-insensitively, consistent with the comparer used by - /// SourceDefinition.Columns and with the field name lookups in this class, so a - /// configuration whose casing differs from the database schema still resolves. + /// Builds the projection used to read the schema of a database object. Returns "*" unless + /// the object holds columns whose data type this provider cannot map to a CLR type, in + /// which case those columns are named out of the projection so the rest stays reachable. + /// The column list comes from the "Columns" schema collection, which reads catalog metadata + /// only and therefore never has to materialize the offending type. /// - /// - /// True when at least one permission reads every field ("fields" absent, "include" absent, - /// or "include": ["*"]). - /// - /// Backing columns explicitly listed in an "include" section. - /// - /// Backing columns excluded from every wildcard permission and never explicitly included. - /// - private sealed record PermittedColumns(bool AllColumns, HashSet Included, HashSet Excluded) - { - /// - /// No column restriction could be derived from the configuration. - /// - public bool IsUnrestricted => AllColumns && Excluded.Count == 0; - - /// - /// Whether the given backing column is readable per the configuration. - /// - public bool IsColumnPermitted(string columnName) - { - if (Included.Contains(columnName)) - { - return true; - } - - if (Excluded.Contains(columnName)) - { - return false; - } - - return AllColumns; - } - } - - /// - /// Resolves the readable columns for a database object. - /// Because the schema DataTable is cached per schema.table, the result combines the - /// permissions of every entity in this data source backed by that same object: a column - /// has to be read if any of those entities can read it. - /// - /// Schema of the database object. - /// Name of the database object. - /// Entity that triggered the schema discovery, when known. - private PermittedColumns ResolvePermittedColumnsForDatabaseObject( - string schemaName, - string tableName, - string? entityName) + private async Task BuildSchemaProjectionAsync(string schemaName, string tableName) { - HashSet included = new(StringComparer.OrdinalIgnoreCase); - HashSet? excluded = null; - bool allColumns = false; - bool matchedAnyEntity = false; - - foreach ((string candidateEntityName, Entity candidateEntity) in Entities) - { - // Only consider entities backed by the same database object. - if (EntityToDatabaseObject.TryGetValue(candidateEntityName, out DatabaseObject? databaseObject)) - { - if (!string.Equals(databaseObject.SchemaName, schemaName, StringComparison.OrdinalIgnoreCase) - || !string.Equals(databaseObject.Name, tableName, StringComparison.OrdinalIgnoreCase)) - { - continue; - } - } - else if (!string.Equals(candidateEntityName, entityName, StringComparison.Ordinal)) - { - // Database object not inferred yet: only the entity that triggered the read applies. - continue; - } - - matchedAnyEntity = true; - PermittedColumns entityPermittedColumns = ResolvePermittedColumnsForEntity(candidateEntity); - - included.UnionWith(entityPermittedColumns.Included); - - if (entityPermittedColumns.AllColumns) - { - allColumns = true; - - // A column is only droppable when every wildcard permission excludes it. - if (excluded is null) - { - excluded = new(entityPermittedColumns.Excluded, StringComparer.OrdinalIgnoreCase); - } - else - { - excluded.IntersectWith(entityPermittedColumns.Excluded); - } - } - } - - if (!matchedAnyEntity) + if (UnsupportedColumnDataTypes.Count == 0) { - return new(AllColumns: true, Included: new(StringComparer.OrdinalIgnoreCase), Excluded: new(StringComparer.OrdinalIgnoreCase)); - } - - excluded ??= new(StringComparer.OrdinalIgnoreCase); - excluded.ExceptWith(included); - - return new(allColumns, included, excluded); - } - - /// - /// Resolves the readable columns of a single entity from the "fields.include" / - /// "fields.exclude" sections of its permissions. Exposed names (mappings and field - /// aliases) are translated back to their database column names, and the configured - /// primary key is always kept, since the runtime cannot operate on the entity without it. - /// - /// - /// The projection is only narrowed for entities whose primary key is known from the - /// configuration ("fields[].primary-key" or "source.key-fields"). When the primary key is - /// instead inferred from the schema read itself - see the fallback to - /// DataTable.PrimaryKey in PopulateObjectDefinitionForEntity - narrowing - /// could drop the key column and leave the entity with no primary key, so every column is - /// read as before. - /// - private static PermittedColumns ResolvePermittedColumnsForEntity(Entity entity) - { - HashSet included = new(StringComparer.OrdinalIgnoreCase); - HashSet? excluded = null; - bool allColumns = false; - - // Exposed name -> backing column name. - Dictionary exposedToBackingName = new(StringComparer.OrdinalIgnoreCase); - - if (entity.Mappings is not null) - { - foreach ((string backingName, string exposedName) in entity.Mappings) - { - if (!string.IsNullOrWhiteSpace(exposedName)) - { - exposedToBackingName[exposedName] = backingName; - } - } - } - - if (entity.Fields is not null) - { - foreach (FieldMetadata field in entity.Fields) - { - if (!string.IsNullOrWhiteSpace(field.Alias)) - { - exposedToBackingName[field.Alias!] = field.Name; - } - } - } - - HashSet configuredPrimaryKey = new(StringComparer.OrdinalIgnoreCase); - - if (entity.Fields is not null) - { - foreach (FieldMetadata field in entity.Fields.Where(f => f.PrimaryKey)) - { - configuredPrimaryKey.Add(ResolveBackingName(field.Name, exposedToBackingName)); - } - } - - if (configuredPrimaryKey.Count == 0 && entity.Source is not null && entity.Source.KeyFields is not null) - { - foreach (string keyField in entity.Source.KeyFields) - { - configuredPrimaryKey.Add(ResolveBackingName(keyField, exposedToBackingName)); - } + return "*"; } - // Without a configured primary key, the key itself is inferred from this schema read, - // so a narrowed projection could drop it and leave the entity unusable. Read it all. - if (entity.Permissions is null || entity.Permissions.Length == 0 || configuredPrimaryKey.Count == 0) - { - return new(AllColumns: true, included, new HashSet(StringComparer.OrdinalIgnoreCase)); - } + List readableColumns = new(); + List skippedColumns = new(); - foreach (EntityPermission permission in entity.Permissions) + try { - if (permission.Actions is null) - { - allColumns = true; - excluded = new(StringComparer.OrdinalIgnoreCase); - continue; - } + DataTable columnsInTable = await GetColumnsAsync(schemaName, tableName); - foreach (EntityAction action in permission.Actions) + foreach (DataRow columnInfo in columnsInTable.Rows) { - EntityActionFields? fields = action.Fields; - - HashSet actionExcluded = new(StringComparer.OrdinalIgnoreCase); - if (fields?.Exclude is not null) + if (columnInfo["COLUMN_NAME"] is not string columnName) { - if (fields.Exclude.Contains(FIELD_WILDCARD)) - { - // This permission reads no field at all, so it contributes no column. - continue; - } - - foreach (string field in fields.Exclude) - { - actionExcluded.Add(ResolveBackingName(field, exposedToBackingName)); - } + continue; } - // No "fields" section, no "include" section, or an explicit wildcard, - // means every column not listed in "exclude" is readable. - bool includesEveryField = fields is null - || fields.Include is null - || fields.Include.Contains(FIELD_WILDCARD); + string? dataType = columnInfo["DATA_TYPE"] as string; - if (includesEveryField) + if (dataType is not null && UnsupportedColumnDataTypes.Contains(dataType)) { - allColumns = true; - - if (excluded is null) - { - excluded = actionExcluded; - } - else - { - excluded.IntersectWith(actionExcluded); - } - - continue; + skippedColumns.Add($"{columnName} ({dataType})"); } - - foreach (string field in fields!.Include!) + else { - string backingName = ResolveBackingName(field, exposedToBackingName); - if (!actionExcluded.Contains(backingName)) - { - included.Add(backingName); - } + readableColumns.Add(columnName); } } } - - // The primary key is structural: never drop it from the projection. - included.UnionWith(configuredPrimaryKey); - - excluded ??= new(StringComparer.OrdinalIgnoreCase); - excluded.ExceptWith(included); - - return new(allColumns, included, excluded); - } - - /// - /// Translates a configured (exposed) field name into its backing column name. - /// - private static string ResolveBackingName(string fieldName, Dictionary exposedToBackingName) - { - return exposedToBackingName.TryGetValue(fieldName, out string? backingName) ? backingName : fieldName; - } - - /// - /// Builds the projection used to read the schema of a database object, narrowed to the - /// columns the configuration allows to be read. Returns "*" when no restriction applies - /// or when the column list cannot be determined. - /// - private async Task BuildSchemaProjectionAsync( - string schemaName, - string tableName, - PermittedColumns permittedColumns) - { - if (permittedColumns.IsUnrestricted) + catch (Exception ex) when (ex is not DataApiBuilderException) { + // The column list is a best-effort optimization: without it the read below behaves + // exactly as it did before, failing loudly if an unsupported type is present. + _logger.LogDebug( + "Unable to enumerate the columns of {schemaName}.{tableName}: {message}", + schemaName, + tableName, + ex.Message); return "*"; } - List columnsToRead; - - if (permittedColumns.AllColumns) - { - // "include": ["*"] with an "exclude" list: enumerate the columns from the catalog - // (metadata only, so unsupported CLR types are never materialized) and drop the - // excluded ones. - List allColumnNames = new(); - try - { - DataTable columnsInTable = await GetColumnsAsync(schemaName, tableName); - - foreach (DataRow columnInfo in columnsInTable.Rows) - { - if (columnInfo["COLUMN_NAME"] is string columnName) - { - allColumnNames.Add(columnName); - } - } - } - catch (Exception ex) - { - _logger.LogDebug( - "Unable to enumerate the columns of {schemaName}.{tableName} to honor the configured field exclusions: {message}", - schemaName, - tableName, - ex.Message); - return "*"; - } - - if (allColumnNames.Count == 0) - { - return "*"; - } - - columnsToRead = allColumnNames.Where(permittedColumns.IsColumnPermitted).ToList(); - } - else - { - columnsToRead = permittedColumns.Included.ToList(); - } - - if (columnsToRead.Count == 0) + if (skippedColumns.Count == 0 || readableColumns.Count == 0) { return "*"; } - return string.Join(", ", columnsToRead.Select(column => SqlQueryBuilder.QuoteIdentifier(column))); + _logger.LogWarning( + "Skipping column(s) of {schemaName}.{tableName} whose data type is not supported: {skippedColumns}. " + + "They are not exposed through REST, GraphQL or MCP.", + schemaName, + tableName, + string.Join(", ", skippedColumns)); + + return string.Join(", ", readableColumns.Select(column => SqlQueryBuilder.QuoteIdentifier(column))); } /// diff --git a/src/Service.Tests/Snapshots/ConfigurationTests.TestReadingRuntimeConfigForMsSql.verified.txt b/src/Service.Tests/Snapshots/ConfigurationTests.TestReadingRuntimeConfigForMsSql.verified.txt index 57e7548405..b0ce94e022 100644 --- a/src/Service.Tests/Snapshots/ConfigurationTests.TestReadingRuntimeConfigForMsSql.verified.txt +++ b/src/Service.Tests/Snapshots/ConfigurationTests.TestReadingRuntimeConfigForMsSql.verified.txt @@ -1929,13 +1929,7 @@ Role: anonymous, Actions: [ { - Action: Read, - Fields: { - Include: [ - id, - name - ] - } + Action: Read } ] } diff --git a/src/Service.Tests/UnitTests/SqlMetadataProviderUnitTests.cs b/src/Service.Tests/UnitTests/SqlMetadataProviderUnitTests.cs index 50ed2026f0..c7dbe7d518 100644 --- a/src/Service.Tests/UnitTests/SqlMetadataProviderUnitTests.cs +++ b/src/Service.Tests/UnitTests/SqlMetadataProviderUnitTests.cs @@ -397,14 +397,15 @@ public async Task ValidateInferredRelationshipInfoForMsSql() } /// - /// Test to validate that a table holding a column whose CLR type the data provider cannot - /// resolve - here a geometry column - is still usable when the entity permissions enumerate - /// the readable fields and that column is not among them. - /// Metadata inference must succeed and the unreadable column must be absent from the - /// inferred source definition, so it never reaches the OData or GraphQL type maps. + /// Test to validate that a table holding a column whose data type the data provider cannot + /// map to a CLR type - here a geometry column - is still usable: metadata inference must + /// succeed and the unsupported column must be absent from the inferred source definition, + /// so it never reaches the OData or GraphQL type maps. + /// The entity places no field restriction, so this covers the column being skipped on the + /// strength of its type alone. /// [TestMethod, TestCategory(TestCategory.MSSQL)] - public async Task ValidateColumnExcludedByFieldPermissionsIsNotInferred() + public async Task ValidateUnsupportedColumnTypeIsNotInferred() { DatabaseEngine = TestCategory.MSSQL; await SetupTestFixtureAndInferMetadata(); @@ -420,10 +421,10 @@ public async Task ValidateColumnExcludedByFieldPermissionsIsNotInferred() message: "The primary key column is expected in the source definition."); Assert.IsTrue( sourceDefinition.Columns.ContainsKey("name"), - message: "A column listed in fields.include is expected in the source definition."); + message: "A column with a supported data type is expected in the source definition."); Assert.IsFalse( sourceDefinition.Columns.ContainsKey("geom"), - message: "A column absent from fields.include is not expected in the source definition."); + message: "A column whose data type cannot be mapped is not expected in the source definition."); TestHelper.UnsetAllDABEnvironmentVariables(); } diff --git a/src/Service.Tests/dab-config.MsSql.json b/src/Service.Tests/dab-config.MsSql.json index fa80aa2e29..fc614d0e86 100644 --- a/src/Service.Tests/dab-config.MsSql.json +++ b/src/Service.Tests/dab-config.MsSql.json @@ -2010,14 +2010,7 @@ "role": "anonymous", "actions": [ { - "action": "read", - "fields": { - "exclude": [], - "include": [ - "id", - "name" - ] - } + "action": "read" } ] } From 0df9ee491b80b554f4795c5eb22bf3171a84a15d Mon Sep 17 00:00:00 2001 From: Joymax Andrade do Nascimento Date: Fri, 11 Sep 2026 14:27:07 -0300 Subject: [PATCH 06/19] Reject unsupported primary keys and read the catalog once per object --- .../MetadataProviders/SqlMetadataProvider.cs | 150 ++++++++++++++++-- 1 file changed, 140 insertions(+), 10 deletions(-) diff --git a/src/Core/Services/MetadataProviders/SqlMetadataProvider.cs b/src/Core/Services/MetadataProviders/SqlMetadataProvider.cs index d801290044..87cb754bac 100644 --- a/src/Core/Services/MetadataProviders/SqlMetadataProvider.cs +++ b/src/Core/Services/MetadataProviders/SqlMetadataProvider.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +using System.Collections.Concurrent; using System.Collections.Immutable; using System.Collections.ObjectModel; using System.Data; @@ -95,6 +96,20 @@ public abstract class SqlMetadataProvider : private Dictionary> EntityExposedNamesToBackingColumnNames { get; } = new(); + /// + /// Caches the "Columns" schema collection per database object for the duration of metadata + /// initialization, so schema discovery and column definition population share one catalog + /// round trip instead of querying twice per object. Cleared once initialization completes. + /// + private readonly ConcurrentDictionary _columnsMetadataCache = new(StringComparer.OrdinalIgnoreCase); + + /// + /// Columns left out of the schema projection per database object, mapped to the data type + /// that made them unreadable. Used to explain the omission when a configured primary key + /// turns out to be one of them. + /// + private readonly ConcurrentDictionary> _skippedColumnsByObject = new(StringComparer.OrdinalIgnoreCase); + protected IAbstractQueryManagerFactory QueryManagerFactory { get; init; } /// @@ -347,7 +362,16 @@ public async Task InitializeAsync() _runtimeConfigValidator.ValidateEntityAndAutoentityConfigurations(runtimeConfig); GenerateDatabaseObjectForEntities(); - await PopulateObjectDefinitionForEntities(); + + try + { + await PopulateObjectDefinitionForEntities(); + } + finally + { + ReleaseCatalogMetadataCaches(); + } + GenerateExposedToBackingColumnMapsForEntities(); // When IsLateConfigured is true we are in a hosted scenario and do not reveal primary key information. @@ -1540,7 +1564,6 @@ private async Task PopulateSourceDefinitionAsync( using DataTableReader reader = new(dataTable); DataTable schemaTable = reader.GetSchemaTable(); RuntimeConfig runtimeConfig = _runtimeConfigProvider.GetConfig(); - foreach (DataRow columnInfoFromAdapter in schemaTable.Rows) { string columnName = columnInfoFromAdapter["ColumnName"].ToString()!; @@ -1582,7 +1605,9 @@ private async Task PopulateSourceDefinitionAsync( sourceDefinition.Columns.TryAdd(columnName, column); } - DataTable columnsInTable = await GetColumnsAsync(schemaName, tableName); + RejectPrimaryKeyOnUnsupportedColumn(schemaName, tableName, sourceDefinition); + + DataTable columnsInTable = await GetCachedColumnsAsync(schemaName, tableName); PopulateColumnDefinitionWithHasDefaultAndDbType( sourceDefinition, @@ -1836,6 +1861,11 @@ private async Task FillSchemaForTableAsync( /// The column list comes from the "Columns" schema collection, which reads catalog metadata /// only and therefore never has to materialize the offending type. /// + /// + /// Thrown when every column of the object has an unsupported data type. Returning "*" there + /// would re-issue the projection that cannot be read, hiding the reason behind the + /// provider's own error. + /// private async Task BuildSchemaProjectionAsync(string schemaName, string tableName) { if (UnsupportedColumnDataTypes.Count == 0) @@ -1844,11 +1874,11 @@ private async Task BuildSchemaProjectionAsync(string schemaName, string } List readableColumns = new(); - List skippedColumns = new(); + Dictionary skippedColumns = new(StringComparer.OrdinalIgnoreCase); try { - DataTable columnsInTable = await GetColumnsAsync(schemaName, tableName); + DataTable columnsInTable = await GetCachedColumnsAsync(schemaName, tableName); foreach (DataRow columnInfo in columnsInTable.Rows) { @@ -1857,11 +1887,16 @@ private async Task BuildSchemaProjectionAsync(string schemaName, string continue; } - string? dataType = columnInfo["DATA_TYPE"] as string; + if (columnInfo["DATA_TYPE"] is not string dataType) + { + // The catalog did not report a usable type name, so this column cannot be + // classified. Leave the projection alone rather than guess. + return "*"; + } - if (dataType is not null && UnsupportedColumnDataTypes.Contains(dataType)) + if (UnsupportedColumnDataTypes.Contains(dataType)) { - skippedColumns.Add($"{columnName} ({dataType})"); + skippedColumns[columnName] = dataType; } else { @@ -1881,21 +1916,116 @@ private async Task BuildSchemaProjectionAsync(string schemaName, string return "*"; } - if (skippedColumns.Count == 0 || readableColumns.Count == 0) + if (skippedColumns.Count == 0) { return "*"; } + if (readableColumns.Count == 0) + { + // Falling back to "*" here would re-issue the very projection that fails, so the + // caller would see the opaque provider error instead of the reason for it. + throw new DataApiBuilderException( + message: $"Every column of {schemaName}.{tableName} has a data type that is not supported: " + + $"{FormatSkippedColumns(skippedColumns)}. The object cannot be exposed.", + statusCode: HttpStatusCode.ServiceUnavailable, + subStatusCode: DataApiBuilderException.SubStatusCodes.ErrorInInitialization); + } + + _skippedColumnsByObject[GetObjectCacheKey(schemaName, tableName)] = skippedColumns; + _logger.LogWarning( "Skipping column(s) of {schemaName}.{tableName} whose data type is not supported: {skippedColumns}. " + "They are not exposed through REST, GraphQL or MCP.", schemaName, tableName, - string.Join(", ", skippedColumns)); + FormatSkippedColumns(skippedColumns)); return string.Join(", ", readableColumns.Select(column => SqlQueryBuilder.QuoteIdentifier(column))); } + /// + /// Fails initialization when a configured primary key names a column that was left out of + /// the projection because its data type is not supported. The key would otherwise stay in + /// while being absent from + /// , and the inconsistency surfaces much later as a + /// lookup failure while building queries, the OpenAPI document or the EDM model. + /// + private void RejectPrimaryKeyOnUnsupportedColumn( + string schemaName, + string tableName, + SourceDefinition sourceDefinition) + { + if (!_skippedColumnsByObject.TryGetValue(GetObjectCacheKey(schemaName, tableName), out Dictionary? skippedColumns)) + { + return; + } + + foreach (string primaryKey in sourceDefinition.PrimaryKey) + { + if (skippedColumns.TryGetValue(primaryKey, out string? dataType)) + { + throw new DataApiBuilderException( + message: $"The primary key column {primaryKey} of {schemaName}.{tableName} has the data type " + + $"{dataType}, which is not supported. A primary key cannot be omitted from the object " + + "metadata, so this object cannot be exposed.", + statusCode: HttpStatusCode.ServiceUnavailable, + subStatusCode: DataApiBuilderException.SubStatusCodes.ErrorInInitialization); + } + } + } + + /// + /// Renders skipped columns as "name (type)" pairs for log and error messages. + /// + private static string FormatSkippedColumns(Dictionary skippedColumns) + { + return string.Join(", ", skippedColumns.Select(entry => $"{entry.Key} ({entry.Value})")); + } + + /// + /// Key used by the per-object metadata caches held during initialization. + /// + private static string GetObjectCacheKey(string schemaName, string tableName) + { + return $"{schemaName}.{tableName}"; + } + + /// + /// Releases the catalog metadata gathered during initialization. Nothing reads these caches + /// once object definitions are populated, and the cached tables are disposable. + /// + private void ReleaseCatalogMetadataCaches() + { + foreach (DataTable columnsInTable in _columnsMetadataCache.Values) + { + columnsInTable.Dispose(); + } + + _columnsMetadataCache.Clear(); + _skippedColumnsByObject.Clear(); + } + + /// + /// Returns the "Columns" schema collection for a database object, reading it from the + /// catalog once per object. Schema discovery and column definition population both need it, + /// and each call opens its own connection. + /// + private async Task GetCachedColumnsAsync(string schemaName, string tableName) + { + string cacheKey = GetObjectCacheKey(schemaName, tableName); + + if (_columnsMetadataCache.TryGetValue(cacheKey, out DataTable? cachedColumns)) + { + return cachedColumns; + } + + DataTable columnsInTable = await GetColumnsAsync(schemaName, tableName); + _columnsMetadataCache[cacheKey] = columnsInTable; + + return columnsInTable; + } + /// /// Gets the correctly formatted table name with schema as prefix, if one exists. /// A schema prefix is simply the correctly formatted and prefixed schema name that From 3207712341c2b589ae6a3db73c9be6c7fceeb6e0 Mon Sep 17 00:00:00 2001 From: Joymax Andrade do Nascimento Date: Fri, 11 Sep 2026 15:15:58 -0300 Subject: [PATCH 07/19] Resolve the schema projection before opening the connection and disambiguate the cache key --- .../MetadataProviders/SqlMetadataProvider.cs | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/src/Core/Services/MetadataProviders/SqlMetadataProvider.cs b/src/Core/Services/MetadataProviders/SqlMetadataProvider.cs index 87cb754bac..9de91717e4 100644 --- a/src/Core/Services/MetadataProviders/SqlMetadataProvider.cs +++ b/src/Core/Services/MetadataProviders/SqlMetadataProvider.cs @@ -1834,20 +1834,22 @@ private async Task FillSchemaForTableAsync( innerException: ex); } + string tableNameWithSchemaPrefix = GetTableNameWithSchemaPrefix(schemaName, tableName); + + // Resolved before the connection below is opened. Reading the catalog uses a connection + // of its own, and nesting that inside an already open one exhausts a small pool: with + // "Max Pool Size=1" the inner open waits for a connection the outer scope still holds. + string projection = await BuildSchemaProjectionAsync(schemaName, tableName); + await conn.OpenAsync(); DataAdapterT adapterForTable = new(); CommandT selectCommand = new() { - Connection = conn + Connection = conn, + CommandText + = $"SELECT {projection} FROM {tableNameWithSchemaPrefix}" }; - - string tableNameWithSchemaPrefix = GetTableNameWithSchemaPrefix(schemaName, tableName); - - string projection = await BuildSchemaProjectionAsync(schemaName, tableName); - - selectCommand.CommandText - = $"SELECT {projection} FROM {tableNameWithSchemaPrefix}"; adapterForTable.SelectCommand = selectCommand; DataTable[] dataTable = adapterForTable.FillSchema(EntitiesDataSet, SchemaType.Source, tableNameWithSchemaPrefix); @@ -1984,11 +1986,14 @@ private static string FormatSkippedColumns(Dictionary skippedCol } /// - /// Key used by the per-object metadata caches held during initialization. + /// Key used by the per-object metadata caches held during initialization. The schema name is + /// length-prefixed rather than joined with a dot, because bracketed identifiers may contain + /// dots: [a.b].[c] and [a].[b.c] are different objects that a "schema.table" + /// key would collide, letting one reuse the other's catalog rows. /// private static string GetObjectCacheKey(string schemaName, string tableName) { - return $"{schemaName}.{tableName}"; + return $"{schemaName.Length}:{schemaName}{tableName}"; } /// From 60ffd4c34133bd647a2112f71a5b2ce6fe8cb5d7 Mon Sep 17 00:00:00 2001 From: Joymax Andrade do Nascimento Date: Fri, 11 Sep 2026 16:36:52 -0300 Subject: [PATCH 08/19] Add MSSQL test for an object whose every column has an unsupported data type --- src/Service.Tests/DatabaseSchema-MsSql.sql | 2 + .../UnitTests/SqlMetadataProviderUnitTests.cs | 55 +++++++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/src/Service.Tests/DatabaseSchema-MsSql.sql b/src/Service.Tests/DatabaseSchema-MsSql.sql index ce63144076..0850e0e551 100644 --- a/src/Service.Tests/DatabaseSchema-MsSql.sql +++ b/src/Service.Tests/DatabaseSchema-MsSql.sql @@ -11,6 +11,7 @@ DROP VIEW IF EXISTS books_view_with_mapping; DROP VIEW IF EXISTS stocks_view_selected; DROP VIEW IF EXISTS books_publishers_view_composite; DROP VIEW IF EXISTS books_publishers_view_composite_insertable; +DROP VIEW IF EXISTS geometry_only_view; DROP PROCEDURE IF EXISTS get_books; DROP PROCEDURE IF EXISTS get_book_by_id; DROP PROCEDURE IF EXISTS get_publisher_by_id; @@ -768,6 +769,7 @@ EXEC('CREATE VIEW books_publishers_view_composite_insertable as SELECT books.id, books.title, publishers.name, books.publisher_id FROM dbo.books,dbo.publishers where publishers.id = books.publisher_id'); +EXEC('CREATE VIEW geometry_only_view AS SELECT geom FROM dbo.geometry_type_table'); EXEC('CREATE PROCEDURE get_book_by_id @id int AS SELECT * FROM dbo.books WHERE id = @id'); diff --git a/src/Service.Tests/UnitTests/SqlMetadataProviderUnitTests.cs b/src/Service.Tests/UnitTests/SqlMetadataProviderUnitTests.cs index c7dbe7d518..f2080b11d3 100644 --- a/src/Service.Tests/UnitTests/SqlMetadataProviderUnitTests.cs +++ b/src/Service.Tests/UnitTests/SqlMetadataProviderUnitTests.cs @@ -429,6 +429,61 @@ public async Task ValidateUnsupportedColumnTypeIsNotInferred() TestHelper.UnsetAllDABEnvironmentVariables(); } + /// + /// Test to validate that a database object whose every column has a data type the data + /// provider cannot map fails initialization with a specific error, rather than falling back + /// to reading every column and surfacing the provider's opaque failure instead of the reason. + /// The entity is declared in an in-memory config rather than in dab-config.MsSql.json, + /// because this object fails by design and every MSSQL fixture initializes every configured + /// entity. + /// + [TestMethod, TestCategory(TestCategory.MSSQL)] + public async Task ValidateObjectWithOnlyUnsupportedColumnsFailsInitialization() + { + DatabaseEngine = TestCategory.MSSQL; + TestHelper.SetupDatabaseEnvironment(DatabaseEngine); + + Dictionary entities = new() + { + { + "GeometryOnlyView", + new Entity( + Source: new("dbo.geometry_only_view", EntitySourceType.View, null, new string[] { "geom" }), + Fields: null, + Rest: new(Enabled: true), + GraphQL: new("GeometryOnlyView", "GeometryOnlyViews", Enabled: true), + Permissions: new EntityPermission[] + { + new(Role: "anonymous", + Actions: new EntityAction[] { new(Action: EntityActionOperation.Read, Fields: null, Policy: null) }) + }, + Relationships: null, + Mappings: null) + } + }; + + RuntimeConfig runtimeConfig = SqlTestHelper.SetupRuntimeConfig() with { Entities = new RuntimeEntities(entities) }; + RuntimeConfigProvider runtimeConfigProvider = TestHelper.GenerateInMemoryRuntimeConfigProvider(runtimeConfig); + SetUpSQLMetadataProvider(runtimeConfigProvider); + await ResetDbStateAsync(); + + try + { + await _sqlMetadataProvider.InitializeAsync(); + Assert.Fail("Expected DataApiBuilderException was not thrown for an object whose every column has an unsupported data type."); + } + catch (DataApiBuilderException ex) + { + Assert.AreEqual(HttpStatusCode.ServiceUnavailable, ex.StatusCode); + Assert.AreEqual(DataApiBuilderException.SubStatusCodes.ErrorInInitialization, ex.SubStatusCode); + Assert.IsTrue( + ex.Message.Contains("has a data type that is not supported"), + message: $"Unexpected exception message: {ex.Message}"); + } + + TestHelper.UnsetAllDABEnvironmentVariables(); + } + /// /// Test to validate successful inference of relationship data based on data provided in the config and the metadata /// collected from the MySql database. From 5d65bda8baa9aa1ccb60e10b477d36758d246797 Mon Sep 17 00:00:00 2001 From: Joymax Andrade do Nascimento Date: Fri, 11 Sep 2026 18:00:38 -0300 Subject: [PATCH 09/19] Compare cached object names ordinally and cover the primary key guard --- .../MetadataProviders/SqlMetadataProvider.cs | 12 +++-- .../UnitTests/SqlMetadataProviderUnitTests.cs | 53 +++++++++++++++++++ 2 files changed, 62 insertions(+), 3 deletions(-) diff --git a/src/Core/Services/MetadataProviders/SqlMetadataProvider.cs b/src/Core/Services/MetadataProviders/SqlMetadataProvider.cs index 9de91717e4..570f6561b5 100644 --- a/src/Core/Services/MetadataProviders/SqlMetadataProvider.cs +++ b/src/Core/Services/MetadataProviders/SqlMetadataProvider.cs @@ -100,15 +100,21 @@ public abstract class SqlMetadataProvider : /// Caches the "Columns" schema collection per database object for the duration of metadata /// initialization, so schema discovery and column definition population share one catalog /// round trip instead of querying twice per object. Cleared once initialization completes. + /// The key is compared with an ordinal comparer: under a case-sensitive collation + /// dbo.Foo and dbo.foo are distinct objects, and aliasing them would serve one + /// object's catalog rows for the other. Case-insensitive collations are unaffected, since + /// they cannot hold both names at once. /// - private readonly ConcurrentDictionary _columnsMetadataCache = new(StringComparer.OrdinalIgnoreCase); + private readonly ConcurrentDictionary _columnsMetadataCache = new(StringComparer.Ordinal); /// /// Columns left out of the schema projection per database object, mapped to the data type /// that made them unreadable. Used to explain the omission when a configured primary key - /// turns out to be one of them. + /// turns out to be one of them. Keyed by object with an ordinal comparer for the reason + /// above; the inner column names stay case-insensitive, matching how this class resolves + /// configured field names against the schema. /// - private readonly ConcurrentDictionary> _skippedColumnsByObject = new(StringComparer.OrdinalIgnoreCase); + private readonly ConcurrentDictionary> _skippedColumnsByObject = new(StringComparer.Ordinal); protected IAbstractQueryManagerFactory QueryManagerFactory { get; init; } diff --git a/src/Service.Tests/UnitTests/SqlMetadataProviderUnitTests.cs b/src/Service.Tests/UnitTests/SqlMetadataProviderUnitTests.cs index f2080b11d3..180299603f 100644 --- a/src/Service.Tests/UnitTests/SqlMetadataProviderUnitTests.cs +++ b/src/Service.Tests/UnitTests/SqlMetadataProviderUnitTests.cs @@ -484,6 +484,59 @@ public async Task ValidateObjectWithOnlyUnsupportedColumnsFailsInitialization() TestHelper.UnsetAllDABEnvironmentVariables(); } + /// + /// Test to validate that a configured primary key naming a column of an unsupported data + /// type fails initialization instead of producing a source definition whose primary key is + /// absent from its columns. The error names both the column and the offending type. + /// The entity is declared in an in-memory config for the same reason as the test above. + /// + [TestMethod, TestCategory(TestCategory.MSSQL)] + public async Task ValidatePrimaryKeyOnUnsupportedColumnFailsInitialization() + { + DatabaseEngine = TestCategory.MSSQL; + TestHelper.SetupDatabaseEnvironment(DatabaseEngine); + + Dictionary entities = new() + { + { + "GeometryKeyed", + new Entity( + Source: new("dbo.geometry_type_table", EntitySourceType.Table, null, new string[] { "geom" }), + Fields: null, + Rest: new(Enabled: true), + GraphQL: new("GeometryKeyed", "GeometryKeyeds", Enabled: true), + Permissions: new EntityPermission[] + { + new(Role: "anonymous", + Actions: new EntityAction[] { new(Action: EntityActionOperation.Read, Fields: null, Policy: null) }) + }, + Relationships: null, + Mappings: null) + } + }; + + RuntimeConfig runtimeConfig = SqlTestHelper.SetupRuntimeConfig() with { Entities = new RuntimeEntities(entities) }; + RuntimeConfigProvider runtimeConfigProvider = TestHelper.GenerateInMemoryRuntimeConfigProvider(runtimeConfig); + SetUpSQLMetadataProvider(runtimeConfigProvider); + await ResetDbStateAsync(); + + try + { + await _sqlMetadataProvider.InitializeAsync(); + Assert.Fail("Expected DataApiBuilderException was not thrown for a primary key of an unsupported data type."); + } + catch (DataApiBuilderException ex) + { + Assert.AreEqual(HttpStatusCode.ServiceUnavailable, ex.StatusCode); + Assert.AreEqual(DataApiBuilderException.SubStatusCodes.ErrorInInitialization, ex.SubStatusCode); + Assert.IsTrue( + ex.Message.Contains("geom") && ex.Message.Contains("geometry"), + message: $"The error is expected to name the column and its data type. Actual message: {ex.Message}"); + } + + TestHelper.UnsetAllDABEnvironmentVariables(); + } + /// /// Test to validate successful inference of relationship data based on data provided in the config and the metadata /// collected from the MySql database. From 9b47156dec132eb99515cab179a682baa658f63e Mon Sep 17 00:00:00 2001 From: Joymax Andrade do Nascimento Date: Mon, 14 Sep 2026 21:38:38 -0300 Subject: [PATCH 10/19] Keep the narrowed projection authoritative and reject stale configured references Schema discovery reproduces the columns SELECT * exposes before subtracting unsupported data types, so HIDDEN period columns stay out of the exposed contract. When the projection is narrowed it reads the shape with CommandBehavior.SchemaOnly instead of DbDataAdapter.FillSchema, whose KeyInfo behaviour reintroduced excluded key columns as hidden reader columns, and takes the primary key from the catalog. Ordinary primary-key inference is untouched when nothing is skipped. Configuration naming a column left out of the projection now fails initialization. Addresses the three review findings on #3802. --- .../MsSqlMetadataProvider.cs | 100 ++++++ .../MetadataProviders/SqlMetadataProvider.cs | 319 +++++++++++++++++- src/Service.Tests/DatabaseSchema-MsSql.sql | 60 ++++ .../UnitTests/SqlMetadataProviderUnitTests.cs | 254 ++++++++++++++ 4 files changed, 729 insertions(+), 4 deletions(-) diff --git a/src/Core/Services/MetadataProviders/MsSqlMetadataProvider.cs b/src/Core/Services/MetadataProviders/MsSqlMetadataProvider.cs index 9176b60566..0a37d7e695 100644 --- a/src/Core/Services/MetadataProviders/MsSqlMetadataProvider.cs +++ b/src/Core/Services/MetadataProviders/MsSqlMetadataProvider.cs @@ -62,6 +62,106 @@ public MsSqlMetadataProvider( /// protected override ImmutableHashSet UnsupportedColumnDataTypes => _unsupportedColumnDataTypes; + /// + protected override async Task GetObjectCatalogMetadataAsync( + string schemaName, + string tableName) + { + string schemaParamName = $"{BaseQueryStructure.PARAM_NAME_PREFIX}param0"; + string tableParamName = $"{BaseQueryStructure.PARAM_NAME_PREFIX}param1"; + + // The object is resolved through object_id() from the schema and the quoted table name, + // the same way PopulateColumnDefinitionsWithReadOnlyFlag resolves it. + // is_hidden marks the period columns of a temporal table declared + // GENERATED ALWAYS ... HIDDEN, which "SELECT *" does not return. key_ordinal is null for + // every column outside the primary key, and 1-based within it. + string query = + "select c.name as COLUMN_NAME, c.is_hidden as IS_HIDDEN, ic.key_ordinal as KEY_ORDINAL " + + "from sys.columns as c " + + "left join sys.indexes as i on i.object_id = c.object_id and i.is_primary_key = 1 " + + "left join sys.index_columns as ic on ic.object_id = c.object_id " + + "and ic.index_id = i.index_id and ic.column_id = c.column_id " + + $"where c.object_id = object_id({schemaParamName}+'.'+{tableParamName});"; + + Dictionary parameters = new() + { + { schemaParamName, new(schemaName, DbType.String) }, + { tableParamName, new(SqlQueryBuilder.QuoteTableNameAsDBConnectionParam(tableName), DbType.String) } + }; + + try + { + return await QueryExecutor.ExecuteQueryAsync( + sqltext: query, + parameters: parameters, + dataReaderHandler: SummarizeObjectCatalogMetadataAsync, + dataSourceName: _dataSourceName); + } + catch (Exception ex) + { + // sys.columns.is_hidden exists from SQL Server 2016 on. Where the catalog cannot + // answer — a dedicated SQL pool, or a login without VIEW DEFINITION — returning null + // leaves the projection at "*", which is exactly the behavior before this change. + _logger.LogDebug( + "Unable to read catalog metadata for {schemaName}.{tableName}: {message}", + schemaName, + tableName, + ex.Message); + + return null; + } + } + + /// + /// Turns the catalog rows read by into + /// . + /// Returns null when the object has no rows: that means it was not found in the catalog, and + /// claiming it has no hidden columns would be a guess. + /// + private async Task SummarizeObjectCatalogMetadataAsync( + DbDataReader reader, + List? args = null) + { + DbResultSet catalogRows = await QueryExecutor.ExtractResultSetFromDbDataReaderAsync(reader); + + if (catalogRows.Rows.Count == 0) + { + return null; + } + + ObjectCatalogMetadata catalogMetadata = new(); + List<(string ColumnName, byte KeyOrdinal)> keyColumns = new(); + + foreach (DbResultSetRow catalogRow in catalogRows.Rows) + { + Dictionary columnInfo = catalogRow.Columns; + + if (columnInfo["COLUMN_NAME"] is not string columnName) + { + continue; + } + + if (columnInfo["IS_HIDDEN"] is bool isHidden && isHidden) + { + catalogMetadata.HiddenColumns.Add(columnName); + } + + if (columnInfo["KEY_ORDINAL"] is byte keyOrdinal && keyOrdinal > 0) + { + keyColumns.Add((columnName, keyOrdinal)); + } + } + + keyColumns.Sort((left, right) => left.KeyOrdinal.CompareTo(right.KeyOrdinal)); + + foreach ((string ColumnName, byte KeyOrdinal) keyColumn in keyColumns) + { + catalogMetadata.PrimaryKeyColumns.Add(keyColumn.ColumnName); + } + + return catalogMetadata; + } + public override string GetDefaultSchemaName() { return "dbo"; diff --git a/src/Core/Services/MetadataProviders/SqlMetadataProvider.cs b/src/Core/Services/MetadataProviders/SqlMetadataProvider.cs index 570f6561b5..1c731c44ec 100644 --- a/src/Core/Services/MetadataProviders/SqlMetadataProvider.cs +++ b/src/Core/Services/MetadataProviders/SqlMetadataProvider.cs @@ -116,6 +116,77 @@ public abstract class SqlMetadataProvider : /// private readonly ConcurrentDictionary> _skippedColumnsByObject = new(StringComparer.Ordinal); + /// + /// Catalog facts per database object that the "Columns" schema collection does not report. + /// Only populated for providers that declare , since + /// only those replace "*" with an explicit projection and therefore need them. Keyed by + /// object with an ordinal comparer, for the reason given on . + /// + private readonly ConcurrentDictionary _objectCatalogMetadataCache = new(StringComparer.Ordinal); + + /// + /// The two catalog facts an explicit schema projection needs and the "Columns" schema + /// collection does not carry: which columns the database hides from "SELECT *", and which + /// columns form the object's own primary key. + /// + protected sealed class ObjectCatalogMetadata + { + /// + /// Columns the database omits from "SELECT *" — for SQL Server, the period columns of a + /// temporal table declared GENERATED ALWAYS ... HIDDEN. Naming them in a projection + /// would expose columns that are invisible today, so they are subtracted before the + /// unsupported data types are. + /// + public HashSet HiddenColumns { get; } = new(StringComparer.OrdinalIgnoreCase); + + /// + /// The columns of the object's own primary key, in key order. Replaces the key discovery + /// the data adapter performs under CommandBehavior.KeyInfo, which cannot be used once + /// the projection is narrowed: the adapter appends key columns the projection left out + /// as hidden reader columns, and an unsupported type among them reintroduces the very + /// failure the narrowing avoids. + /// + public List PrimaryKeyColumns { get; } = new(); + } + + /// + /// Reads for a database object. Returns null by default: + /// a provider only implements this when it declares . + /// When it returns null the projection stays "*" and schema discovery behaves exactly as it + /// did before. + /// + protected virtual Task GetObjectCatalogMetadataAsync( + string schemaName, + string tableName) + { + return Task.FromResult(null); + } + + /// + /// Returns for a database object, reading the catalog + /// once per object for the duration of metadata initialization. + /// + private async Task GetCachedObjectCatalogMetadataAsync( + string schemaName, + string tableName) + { + string cacheKey = GetObjectCacheKey(schemaName, tableName); + + if (_objectCatalogMetadataCache.TryGetValue(cacheKey, out ObjectCatalogMetadata? cachedMetadata)) + { + return cachedMetadata; + } + + ObjectCatalogMetadata? catalogMetadata = await GetObjectCatalogMetadataAsync(schemaName, tableName); + + if (catalogMetadata is not null) + { + _objectCatalogMetadataCache[cacheKey] = catalogMetadata; + } + + return catalogMetadata; + } + protected IAbstractQueryManagerFactory QueryManagerFactory { get; init; } /// @@ -1554,6 +1625,10 @@ private async Task PopulateSourceDefinitionAsync( if (sourceDefinition.PrimaryKey.Count == 0) { + // When the object's own primary key is unreadable, say so. The message below reads + // as a configuration mistake, and no configuration can express that key. + RejectUnreadablePrimaryKey(schemaName, tableName); + throw new DataApiBuilderException( message: $"Primary key not configured on the given database object {tableName}", statusCode: HttpStatusCode.ServiceUnavailable, @@ -1613,6 +1688,8 @@ private async Task PopulateSourceDefinitionAsync( RejectPrimaryKeyOnUnsupportedColumn(schemaName, tableName, sourceDefinition); + RejectConfiguredReferencesToSkippedColumns(entityName, entity, schemaName, tableName); + DataTable columnsInTable = await GetCachedColumnsAsync(schemaName, tableName); PopulateColumnDefinitionWithHasDefaultAndDbType( @@ -1849,12 +1926,32 @@ private async Task FillSchemaForTableAsync( await conn.OpenAsync(); + string selectStatement = $"SELECT {projection} FROM {tableNameWithSchemaPrefix}"; + + if (!string.Equals(projection, "*", StringComparison.Ordinal)) + { + // The projection left columns out, so the data adapter cannot be used here: + // FillSchema runs with CommandBehavior.KeyInfo, under which the provider performs + // its own key discovery and appends key columns missing from the SELECT list as + // hidden reader columns. A column whose CLR type the provider cannot resolve + // reintroduces "DataReader.GetFieldType(N) returned null" that way even though the + // projection excluded it — reachable through the object's own primary key, and + // through any unique index, including when a supported key is configured through + // source.key-fields. Reading the shape without KeyInfo keeps the projection + // authoritative; the primary key comes from the catalog instead. + return await ReadSchemaWithoutKeyInfoAsync( + conn, + selectStatement, + tableNameWithSchemaPrefix, + schemaName, + tableName); + } + DataAdapterT adapterForTable = new(); CommandT selectCommand = new() { Connection = conn, - CommandText - = $"SELECT {projection} FROM {tableNameWithSchemaPrefix}" + CommandText = selectStatement }; adapterForTable.SelectCommand = selectCommand; @@ -1862,6 +1959,88 @@ private async Task FillSchemaForTableAsync( return dataTable[0]; } + /// + /// Reads the shape of a narrowed projection without CommandBehavior.KeyInfo and registers + /// the result in under the name the data adapter would have + /// used, so callers find it there on subsequent lookups. The primary key is taken from the + /// catalog, because the adapter's own key discovery is precisely what has to be avoided. + /// + private async Task ReadSchemaWithoutKeyInfoAsync( + ConnectionT conn, + string selectStatement, + string tableNameWithSchemaPrefix, + string schemaName, + string tableName) + { + DataTable dataTable = new(tableNameWithSchemaPrefix); + + using (CommandT selectCommand = new()) + { + selectCommand.Connection = conn; + selectCommand.CommandText = selectStatement; + + // SchemaOnly describes the statement without returning rows. Without KeyInfo the + // reader carries exactly the projected columns and nothing else. + using DbDataReader reader = + await selectCommand.ExecuteReaderAsync(CommandBehavior.SchemaOnly); + + using DataTable? schemaTable = reader.GetSchemaTable(); + + if (schemaTable is null) + { + throw new DataApiBuilderException( + message: $"The data provider reported no schema for {schemaName}.{tableName}.", + statusCode: HttpStatusCode.ServiceUnavailable, + subStatusCode: DataApiBuilderException.SubStatusCodes.ErrorInInitialization); + } + + foreach (DataRow columnInfo in schemaTable.Rows) + { + if (columnInfo["ColumnName"] is not string columnName) + { + continue; + } + + DataColumn column = new(columnName, (Type)columnInfo["DataType"]) + { + AllowDBNull = columnInfo["AllowDBNull"] is bool allowDbNull && allowDbNull, + AutoIncrement = columnInfo["IsAutoIncrement"] is bool isAutoIncrement && isAutoIncrement + }; + + dataTable.Columns.Add(column); + } + } + + ObjectCatalogMetadata? catalogMetadata = + await GetCachedObjectCatalogMetadataAsync(schemaName, tableName); + + if (catalogMetadata is not null && catalogMetadata.PrimaryKeyColumns.Count > 0) + { + List keyColumns = new(); + + foreach (string primaryKeyColumn in catalogMetadata.PrimaryKeyColumns) + { + if (dataTable.Columns.Contains(primaryKeyColumn)) + { + keyColumns.Add(dataTable.Columns[primaryKeyColumn]!); + } + } + + // A key column left out of the projection cannot be reported as a key here, and the + // key is not silently dropped either: PopulateSourceDefinitionAsync fails through + // RejectUnreadablePrimaryKey when that key is the one in effect, while a key + // configured through source.key-fields takes precedence over this one anyway. + if (keyColumns.Count == catalogMetadata.PrimaryKeyColumns.Count) + { + dataTable.PrimaryKey = keyColumns.ToArray(); + } + } + + EntitiesDataSet.Tables.Add(dataTable); + + return dataTable; + } + /// /// Builds the projection used to read the schema of a database object. Returns "*" unless /// the object holds columns whose data type this provider cannot map to a CLR type, in @@ -1886,6 +2065,17 @@ private async Task BuildSchemaProjectionAsync(string schemaName, string try { + ObjectCatalogMetadata? catalogMetadata = + await GetCachedObjectCatalogMetadataAsync(schemaName, tableName); + + if (catalogMetadata is null) + { + // Without the catalog there is no way to tell which columns the database hides + // from "SELECT *". Naming columns anyway would add the hidden period columns of + // a temporal table to the exposed contract, so leave the projection alone. + return "*"; + } + DataTable columnsInTable = await GetCachedColumnsAsync(schemaName, tableName); foreach (DataRow columnInfo in columnsInTable.Rows) @@ -1905,11 +2095,20 @@ private async Task BuildSchemaProjectionAsync(string schemaName, string if (UnsupportedColumnDataTypes.Contains(dataType)) { skippedColumns[columnName] = dataType; + continue; } - else + + if (catalogMetadata.HiddenColumns.Contains(columnName)) { - readableColumns.Add(columnName); + // "SELECT *" does not return this column, so naming it would widen the + // exposed contract instead of preserving it. It is not "skipped": it was + // never part of the object's shape as the engine sees it, and the read-only + // classification does not recognize generated-always period columns, so a + // PUT would try to null them. + continue; } + + readableColumns.Add(columnName); } } catch (Exception ex) when (ex is not DataApiBuilderException) @@ -1983,6 +2182,117 @@ private void RejectPrimaryKeyOnUnsupportedColumn( } } + /// + /// Fails initialization with the reason when the object's own primary key includes a column + /// left out of the projection because its data type is not supported. Without this the + /// caller reports a missing primary key, which reads as something the user forgot to + /// configure even though no configuration can express that key. + /// + private void RejectUnreadablePrimaryKey(string schemaName, string tableName) + { + string cacheKey = GetObjectCacheKey(schemaName, tableName); + + if (!_skippedColumnsByObject.TryGetValue(cacheKey, out Dictionary? skippedColumns) + || !_objectCatalogMetadataCache.TryGetValue(cacheKey, out ObjectCatalogMetadata? catalogMetadata)) + { + return; + } + + foreach (string primaryKeyColumn in catalogMetadata.PrimaryKeyColumns) + { + if (skippedColumns.TryGetValue(primaryKeyColumn, out string? dataType)) + { + throw new DataApiBuilderException( + message: $"The primary key of {schemaName}.{tableName} includes the column {primaryKeyColumn}, " + + $"whose data type {dataType} is not supported. The object cannot be exposed through that " + + "key. Configure source.key-fields with a supported column that identifies a row uniquely, " + + "if the object has one.", + statusCode: HttpStatusCode.ServiceUnavailable, + subStatusCode: DataApiBuilderException.SubStatusCodes.ErrorInInitialization); + } + } + } + + /// + /// Fails initialization when the configuration names a column that was left out of the + /// projection because its data type is not supported. + /// Such a name keeps resolving after the column is gone: the exposed and backing column maps + /// are built from entity fields and mappings without requiring the column to exist in + /// , and the authorization resolver accepts explicitly + /// included field names. The reference then reaches code that indexes + /// and fails per request instead of at startup — a + /// permission-derived default projection selects the column and the read fails serializing + /// it, a mutation throws while resolving the backing column, and MCP metadata advertises a + /// field no other surface has. Rejecting here keeps the configuration and the exposed + /// contract in agreement. Nothing that worked before stops working: an object with such a + /// column failed discovery outright, so the entity never loaded. + /// + private void RejectConfiguredReferencesToSkippedColumns( + string entityName, + Entity? entity, + string schemaName, + string tableName) + { + if (entity is null + || !_skippedColumnsByObject.TryGetValue( + GetObjectCacheKey(schemaName, tableName), + out Dictionary? skippedColumns)) + { + return; + } + + // "mappings" and "fields" both key on the backing column name. + if (entity.Mappings is not null) + { + foreach (string backingColumn in entity.Mappings.Keys) + { + RejectReference(backingColumn, "mappings"); + } + } + + if (entity.Fields is not null) + { + foreach (FieldMetadata field in entity.Fields) + { + RejectReference(field.Name, "fields"); + } + } + + foreach (EntityPermission permission in entity.Permissions) + { + foreach (EntityAction action in permission.Actions) + { + if (action.Fields?.Include is null) + { + continue; + } + + // An include list names columns that have to be readable. An exclude list naming + // one of these columns asks for what already happened, so it is left alone. + foreach (string includedField in action.Fields.Include) + { + RejectReference(includedField, $"permissions of role {permission.Role}"); + } + } + } + + void RejectReference(string configuredName, string configurationSection) + { + if (!skippedColumns.TryGetValue(configuredName, out string? dataType)) + { + return; + } + + throw new DataApiBuilderException( + message: $"The {configurationSection} of entity {entityName} reference the column {configuredName} " + + $"of {schemaName}.{tableName}, whose data type {dataType} is not supported. That column is not " + + "part of the exposed contract, so the reference cannot be honored. Remove it from the " + + "configuration.", + statusCode: HttpStatusCode.ServiceUnavailable, + subStatusCode: DataApiBuilderException.SubStatusCodes.ErrorInInitialization); + } + } + /// /// Renders skipped columns as "name (type)" pairs for log and error messages. /// @@ -2015,6 +2325,7 @@ private void ReleaseCatalogMetadataCaches() _columnsMetadataCache.Clear(); _skippedColumnsByObject.Clear(); + _objectCatalogMetadataCache.Clear(); } /// diff --git a/src/Service.Tests/DatabaseSchema-MsSql.sql b/src/Service.Tests/DatabaseSchema-MsSql.sql index 0850e0e551..d94ef77c48 100644 --- a/src/Service.Tests/DatabaseSchema-MsSql.sql +++ b/src/Service.Tests/DatabaseSchema-MsSql.sql @@ -46,6 +46,15 @@ DROP TABLE IF EXISTS type_table; DROP TABLE IF EXISTS vector_type_table; DROP TABLE IF EXISTS vector_owners; DROP TABLE IF EXISTS geometry_type_table; +DROP TABLE IF EXISTS hierarchyid_pk_table; +DROP TABLE IF EXISTS hierarchyid_composite_pk_table; +DROP TABLE IF EXISTS hierarchyid_unique_table; +-- System versioning has to be released before the temporal table can be dropped. +IF OBJECT_ID('dbo.temporal_geometry_type_table', 'U') IS NOT NULL + AND OBJECTPROPERTY(OBJECT_ID('dbo.temporal_geometry_type_table'), 'TableTemporalType') = 2 + ALTER TABLE temporal_geometry_type_table SET (SYSTEM_VERSIONING = OFF); +DROP TABLE IF EXISTS temporal_geometry_type_table; +DROP TABLE IF EXISTS temporal_geometry_type_table_history; DROP TABLE IF EXISTS profiles; DROP TABLE IF EXISTS trees; DROP TABLE IF EXISTS fungi; @@ -260,6 +269,43 @@ CREATE TABLE geometry_type_table( geom geometry NULL ); +-- The period columns are HIDDEN, so SELECT * does not return them. An explicit projection built +-- from the catalog would, which is what this fixture guards against. +CREATE TABLE temporal_geometry_type_table( + id int NOT NULL PRIMARY KEY, + name varchar(100) NOT NULL, + geom geometry NULL, + valid_from datetime2 GENERATED ALWAYS AS ROW START HIDDEN NOT NULL, + valid_to datetime2 GENERATED ALWAYS AS ROW END HIDDEN NOT NULL, + PERIOD FOR SYSTEM_TIME (valid_from, valid_to) +) +WITH (SYSTEM_VERSIONING = ON (HISTORY_TABLE = dbo.temporal_geometry_type_table_history)); + +-- The database primary key is itself of an unsupported type, so no projection can expose the +-- object: the engine needs the key it cannot read. +CREATE TABLE hierarchyid_pk_table( + node hierarchyid NOT NULL PRIMARY KEY, + name varchar(100) NOT NULL +); + +-- Same, with the unsupported column as one member of a composite key. +CREATE TABLE hierarchyid_composite_pk_table( + tenant_id int NOT NULL, + node hierarchyid NOT NULL, + name varchar(100) NOT NULL, + CONSTRAINT PK_hierarchyid_composite_pk_table PRIMARY KEY (tenant_id, node) +); + +-- No database primary key, and the unsupported column carries a unique index. This object is +-- expected to load, with the key supplied through source.key-fields: the data adapter's own key +-- discovery would otherwise pull the unique column back in as a hidden reader column. +CREATE TABLE hierarchyid_unique_table( + id int NOT NULL, + node hierarchyid NOT NULL, + name varchar(100) NOT NULL, + CONSTRAINT UQ_hierarchyid_unique_table_node UNIQUE (node) +); + CREATE TABLE profiles( id int IDENTITY(5001, 1) PRIMARY KEY, metadata json NULL @@ -671,6 +717,20 @@ VALUES (2, 'null geometry', NULL); SET IDENTITY_INSERT geometry_type_table OFF +INSERT INTO temporal_geometry_type_table(id, name, geom) +VALUES + (1, 'point', geometry::STGeomFromText('POINT(1 2)', 0)), + (2, 'null geometry', NULL); + +INSERT INTO hierarchyid_pk_table(node, name) +VALUES (hierarchyid::Parse('/1/'), 'root'); + +INSERT INTO hierarchyid_composite_pk_table(tenant_id, node, name) +VALUES (1, hierarchyid::Parse('/1/'), 'root'); + +INSERT INTO hierarchyid_unique_table(id, node, name) +VALUES (1, hierarchyid::Parse('/1/'), 'root'); + SET IDENTITY_INSERT profiles ON INSERT INTO profiles(id, metadata) VALUES diff --git a/src/Service.Tests/UnitTests/SqlMetadataProviderUnitTests.cs b/src/Service.Tests/UnitTests/SqlMetadataProviderUnitTests.cs index 180299603f..68f56a80c2 100644 --- a/src/Service.Tests/UnitTests/SqlMetadataProviderUnitTests.cs +++ b/src/Service.Tests/UnitTests/SqlMetadataProviderUnitTests.cs @@ -537,6 +537,260 @@ public async Task ValidatePrimaryKeyOnUnsupportedColumnFailsInitialization() TestHelper.UnsetAllDABEnvironmentVariables(); } + /// + /// Test to validate that the period columns of a temporal table declared + /// GENERATED ALWAYS ... HIDDEN stay out of the inferred source definition. + /// "SELECT *" does not return them, so an explicit projection built from the catalog must not + /// name them either: adding them widens the exposed contract, and the read-only + /// classification does not recognize generated-always period columns, so an overwriting PUT + /// would try to null them and fail. + /// + [TestMethod, TestCategory(TestCategory.MSSQL)] + public async Task ValidateHiddenPeriodColumnsAreNotInferred() + { + DatabaseEngine = TestCategory.MSSQL; + TestHelper.SetupDatabaseEnvironment(DatabaseEngine); + + await SetUpSingleEntityMetadataProviderAsync( + "TemporalGeometryType", + BuildReadOnlyEntity( + entityName: "TemporalGeometryType", + databaseObject: "dbo.temporal_geometry_type_table", + sourceType: EntitySourceType.Table, + keyFields: new string[] { "id" })); + + await _sqlMetadataProvider.InitializeAsync(); + + Assert.IsTrue( + _sqlMetadataProvider.GetEntityNamesAndDbObjects().TryGetValue("TemporalGeometryType", out DatabaseObject databaseObject), + message: "Metadata inference failed for the entity backed by a temporal table."); + + SourceDefinition sourceDefinition = databaseObject.SourceDefinition; + + Assert.IsTrue( + sourceDefinition.Columns.ContainsKey("id"), + message: "The configured key column is expected in the source definition."); + Assert.IsTrue( + sourceDefinition.Columns.ContainsKey("name"), + message: "A column with a supported data type is expected in the source definition."); + Assert.IsFalse( + sourceDefinition.Columns.ContainsKey("geom"), + message: "A column whose data type cannot be mapped is not expected in the source definition."); + Assert.IsFalse( + sourceDefinition.Columns.ContainsKey("valid_from"), + message: "A HIDDEN period column is not returned by SELECT * and is not expected in the source definition."); + Assert.IsFalse( + sourceDefinition.Columns.ContainsKey("valid_to"), + message: "A HIDDEN period column is not returned by SELECT * and is not expected in the source definition."); + + TestHelper.UnsetAllDABEnvironmentVariables(); + } + + /// + /// Test to validate that an object whose own database primary key is a column of an + /// unsupported data type fails initialization with the reason. + /// The projection cannot carry that column, and the engine cannot operate on the object + /// without its key, so the object is unreachable either way. What this asserts is that the + /// failure names the column and its type instead of reporting a missing primary key, which + /// reads as something the user forgot to configure. + /// + [TestMethod, TestCategory(TestCategory.MSSQL)] + public async Task ValidateUnsupportedDatabasePrimaryKeyFailsInitialization() + { + DatabaseEngine = TestCategory.MSSQL; + TestHelper.SetupDatabaseEnvironment(DatabaseEngine); + + await SetUpSingleEntityMetadataProviderAsync( + "HierarchyIdKeyed", + BuildReadOnlyEntity( + entityName: "HierarchyIdKeyed", + databaseObject: "dbo.hierarchyid_pk_table", + sourceType: EntitySourceType.Table, + keyFields: null)); + + try + { + await _sqlMetadataProvider.InitializeAsync(); + Assert.Fail("Expected DataApiBuilderException was not thrown for a database primary key of an unsupported data type."); + } + catch (DataApiBuilderException ex) + { + Assert.AreEqual(HttpStatusCode.ServiceUnavailable, ex.StatusCode); + Assert.AreEqual(DataApiBuilderException.SubStatusCodes.ErrorInInitialization, ex.SubStatusCode); + Assert.IsTrue( + ex.Message.Contains("node") && ex.Message.Contains("hierarchyid"), + message: $"The error is expected to name the key column and its data type. Actual message: {ex.Message}"); + } + + TestHelper.UnsetAllDABEnvironmentVariables(); + } + + /// + /// Test to validate the same rejection when the unsupported column is one member of a + /// composite database primary key. The supported member alone does not identify a row, so + /// the object cannot be exposed through a partial key. + /// + [TestMethod, TestCategory(TestCategory.MSSQL)] + public async Task ValidateUnsupportedColumnInCompositeDatabasePrimaryKeyFailsInitialization() + { + DatabaseEngine = TestCategory.MSSQL; + TestHelper.SetupDatabaseEnvironment(DatabaseEngine); + + await SetUpSingleEntityMetadataProviderAsync( + "HierarchyIdCompositeKeyed", + BuildReadOnlyEntity( + entityName: "HierarchyIdCompositeKeyed", + databaseObject: "dbo.hierarchyid_composite_pk_table", + sourceType: EntitySourceType.Table, + keyFields: null)); + + try + { + await _sqlMetadataProvider.InitializeAsync(); + Assert.Fail("Expected DataApiBuilderException was not thrown for a composite database primary key holding an unsupported data type."); + } + catch (DataApiBuilderException ex) + { + Assert.AreEqual(HttpStatusCode.ServiceUnavailable, ex.StatusCode); + Assert.AreEqual(DataApiBuilderException.SubStatusCodes.ErrorInInitialization, ex.SubStatusCode); + Assert.IsTrue( + ex.Message.Contains("node") && ex.Message.Contains("hierarchyid"), + message: $"The error is expected to name the key column and its data type. Actual message: {ex.Message}"); + } + + TestHelper.UnsetAllDABEnvironmentVariables(); + } + + /// + /// Test to validate that an object carrying a unique index over a column of an unsupported + /// data type loads when a supported key is configured through source.key-fields. + /// This is the case the data adapter cannot serve: FillSchema runs with + /// CommandBehavior.KeyInfo, under which the provider performs its own key discovery and + /// appends key columns missing from the SELECT list as hidden reader columns - so the + /// unsupported unique column comes back regardless of the projection, and configuring a + /// supported key does not change that. Schema discovery therefore reads the shape without + /// KeyInfo once the projection is narrowed, and takes the primary key from the catalog. + /// + [TestMethod, TestCategory(TestCategory.MSSQL)] + public async Task ValidateConfiguredKeyIsUsedWhenUniqueIndexColumnIsUnsupported() + { + DatabaseEngine = TestCategory.MSSQL; + TestHelper.SetupDatabaseEnvironment(DatabaseEngine); + + await SetUpSingleEntityMetadataProviderAsync( + "HierarchyIdUnique", + BuildReadOnlyEntity( + entityName: "HierarchyIdUnique", + databaseObject: "dbo.hierarchyid_unique_table", + sourceType: EntitySourceType.Table, + keyFields: new string[] { "id" })); + + await _sqlMetadataProvider.InitializeAsync(); + + Assert.IsTrue( + _sqlMetadataProvider.GetEntityNamesAndDbObjects().TryGetValue("HierarchyIdUnique", out DatabaseObject databaseObject), + message: "Metadata inference failed for an object whose unique index covers an unsupported column."); + + SourceDefinition sourceDefinition = databaseObject.SourceDefinition; + + Assert.IsTrue( + sourceDefinition.Columns.ContainsKey("id"), + message: "The configured key column is expected in the source definition."); + Assert.IsTrue( + sourceDefinition.Columns.ContainsKey("name"), + message: "A column with a supported data type is expected in the source definition."); + Assert.IsFalse( + sourceDefinition.Columns.ContainsKey("node"), + message: "The unsupported unique column is not expected in the source definition."); + CollectionAssert.AreEqual( + new List { "id" }, + sourceDefinition.PrimaryKey, + message: "The configured key is expected to be the primary key in effect."); + + TestHelper.UnsetAllDABEnvironmentVariables(); + } + + /// + /// Test to validate that configuration naming a column left out of the projection fails + /// initialization. + /// Such a name keeps resolving after the column is gone, because the exposed and backing + /// column maps are built from entity fields and mappings without requiring the column to + /// exist in the source definition. The reference then reaches code that indexes the source + /// definition columns and fails per request rather than at startup. + /// + [TestMethod, TestCategory(TestCategory.MSSQL)] + public async Task ValidateConfiguredReferenceToUnsupportedColumnFailsInitialization() + { + DatabaseEngine = TestCategory.MSSQL; + TestHelper.SetupDatabaseEnvironment(DatabaseEngine); + + await SetUpSingleEntityMetadataProviderAsync( + "GeometryAliased", + BuildReadOnlyEntity( + entityName: "GeometryAliased", + databaseObject: "dbo.geometry_type_table", + sourceType: EntitySourceType.Table, + keyFields: new string[] { "id" }, + mappings: new Dictionary { { "geom", "Position" } })); + + try + { + await _sqlMetadataProvider.InitializeAsync(); + Assert.Fail("Expected DataApiBuilderException was not thrown for a mapping over a column of an unsupported data type."); + } + catch (DataApiBuilderException ex) + { + Assert.AreEqual(HttpStatusCode.ServiceUnavailable, ex.StatusCode); + Assert.AreEqual(DataApiBuilderException.SubStatusCodes.ErrorInInitialization, ex.SubStatusCode); + Assert.IsTrue( + ex.Message.Contains("geom") && ex.Message.Contains("mappings"), + message: $"The error is expected to name the column and the configuration section referencing it. Actual message: {ex.Message}"); + } + + TestHelper.UnsetAllDABEnvironmentVariables(); + } + + /// + /// Builds a metadata provider over a single in-memory entity and resets the database state. + /// The objects exercised by the unsupported-data-type tests are declared in memory rather + /// than in dab-config.MsSql.json, because several of them fail by design and every MSSQL + /// fixture initializes every configured entity. + /// + private static async Task SetUpSingleEntityMetadataProviderAsync(string entityName, Entity entity) + { + RuntimeConfig runtimeConfig = SqlTestHelper.SetupRuntimeConfig() + with + { Entities = new RuntimeEntities(new Dictionary { { entityName, entity } }) }; + RuntimeConfigProvider runtimeConfigProvider = TestHelper.GenerateInMemoryRuntimeConfigProvider(runtimeConfig); + SetUpSQLMetadataProvider(runtimeConfigProvider); + await ResetDbStateAsync(); + } + + /// + /// Builds a read-only entity over a database object, with optional configured key fields and + /// mappings. + /// + private static Entity BuildReadOnlyEntity( + string entityName, + string databaseObject, + EntitySourceType sourceType, + string[] keyFields, + Dictionary mappings = null) + { + return new Entity( + Source: new(databaseObject, sourceType, null, keyFields), + Fields: null, + Rest: new(Enabled: true), + GraphQL: new(entityName, $"{entityName}s", Enabled: true), + Permissions: new EntityPermission[] + { + new(Role: "anonymous", + Actions: new EntityAction[] { new(Action: EntityActionOperation.Read, Fields: null, Policy: null) }) + }, + Relationships: null, + Mappings: mappings); + } + /// /// Test to validate successful inference of relationship data based on data provided in the config and the metadata /// collected from the MySql database. From fc0e094e93a9228a2deb375a2acb6d97cf42d4d6 Mon Sep 17 00:00:00 2001 From: Joymax Andrade do Nascimento Date: Mon, 14 Sep 2026 22:37:33 -0300 Subject: [PATCH 11/19] Correct identifier delimiting, identity typing and key inference on the narrowed path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit object_id() now delimits both name parts through QUOTENAME, so a schema or table needing quoting resolves instead of returning no rows and silently leaving the projection at SELECT *. Identity no longer travels through DataColumn.AutoIncrement, whose setter coerces a DataType it cannot increment to Int32 — SQL Server allows identity on tinyint, numeric and decimal. It comes from sys.columns.is_identity and is applied after the columns are populated, so the provider-reported DataType is preserved. Absent a database primary key, the first non-nullable unique index whose key columns are all readable is inferred, in index order. The data adapter does this on the unnarrowed path, and dropping it made the narrowed path demand source.key-fields for objects the other path resolves on its own. The catalog read is deferred until an object is known to hold an unsupported type, so objects that need no narrowing no longer pay an extra query per startup. Adds unique_key_geometry_table and ValidateUniqueKeyIsInferredWhenNoDatabasePrimaryKeyExists. Addresses the Copilot review on #3802. --- .../MsSqlMetadataProvider.cs | 101 +++++++- .../MetadataProviders/SqlMetadataProvider.cs | 242 ++++++++++++++---- src/Service.Tests/DatabaseSchema-MsSql.sql | 16 ++ .../UnitTests/SqlMetadataProviderUnitTests.cs | 43 ++++ 4 files changed, 336 insertions(+), 66 deletions(-) diff --git a/src/Core/Services/MetadataProviders/MsSqlMetadataProvider.cs b/src/Core/Services/MetadataProviders/MsSqlMetadataProvider.cs index 0a37d7e695..cde419fd6e 100644 --- a/src/Core/Services/MetadataProviders/MsSqlMetadataProvider.cs +++ b/src/Core/Services/MetadataProviders/MsSqlMetadataProvider.cs @@ -70,23 +70,36 @@ public MsSqlMetadataProvider( string schemaParamName = $"{BaseQueryStructure.PARAM_NAME_PREFIX}param0"; string tableParamName = $"{BaseQueryStructure.PARAM_NAME_PREFIX}param1"; - // The object is resolved through object_id() from the schema and the quoted table name, - // the same way PopulateColumnDefinitionsWithReadOnlyFlag resolves it. + // The object is resolved through object_id(). Both name parts go through QUOTENAME: + // object_id() parses its argument as a multi-part name, so an unquoted schema or table + // holding a dot, a space, a reserved word or a closing bracket resolves to the wrong + // object or to null — and a null object_id returns no rows, which would leave the + // projection at "SELECT *" and bring #3801 back for that object. QUOTENAME also doubles + // an embedded "]", so the names are passed raw and quoted by the server. // is_hidden marks the period columns of a temporal table declared // GENERATED ALWAYS ... HIDDEN, which "SELECT *" does not return. key_ordinal is null for - // every column outside the primary key, and 1-based within it. + // every column outside the primary key, and 1-based within it; an index's included + // columns report 0 and are not part of the key. + // Unique indexes are returned alongside the primary key, because absent a primary key + // the data adapter reports a non-nullable unique key as DataTable.PrimaryKey on the + // unnarrowed path, and the narrowed path has to do the same. Filtered and disabled + // indexes do not identify every row, and an index's included columns report key_ordinal + // 0 and are not part of its key. string query = - "select c.name as COLUMN_NAME, c.is_hidden as IS_HIDDEN, ic.key_ordinal as KEY_ORDINAL " + "select c.name as COLUMN_NAME, c.is_hidden as IS_HIDDEN, c.is_identity as IS_IDENTITY, " + + "c.is_nullable as IS_NULLABLE, i.index_id as INDEX_ID, " + + "i.is_primary_key as IS_PRIMARY_KEY, ic.key_ordinal as KEY_ORDINAL " + "from sys.columns as c " - + "left join sys.indexes as i on i.object_id = c.object_id and i.is_primary_key = 1 " + "left join sys.index_columns as ic on ic.object_id = c.object_id " - + "and ic.index_id = i.index_id and ic.column_id = c.column_id " - + $"where c.object_id = object_id({schemaParamName}+'.'+{tableParamName});"; + + "and ic.column_id = c.column_id and ic.key_ordinal > 0 " + + "left join sys.indexes as i on i.object_id = ic.object_id and i.index_id = ic.index_id " + + "and i.is_unique = 1 and i.is_disabled = 0 and i.has_filter = 0 " + + $"where c.object_id = object_id(quotename({schemaParamName})+'.'+quotename({tableParamName}));"; Dictionary parameters = new() { { schemaParamName, new(schemaName, DbType.String) }, - { tableParamName, new(SqlQueryBuilder.QuoteTableNameAsDBConnectionParam(tableName), DbType.String) } + { tableParamName, new(tableName, DbType.String) } }; try @@ -130,7 +143,8 @@ public MsSqlMetadataProvider( } ObjectCatalogMetadata catalogMetadata = new(); - List<(string ColumnName, byte KeyOrdinal)> keyColumns = new(); + Dictionary nullabilityByColumn = new(StringComparer.OrdinalIgnoreCase); + Dictionary indexKeysByIndexId = new(); foreach (DbResultSetRow catalogRow in catalogRows.Rows) { @@ -146,22 +160,81 @@ public MsSqlMetadataProvider( catalogMetadata.HiddenColumns.Add(columnName); } - if (columnInfo["KEY_ORDINAL"] is byte keyOrdinal && keyOrdinal > 0) + if (columnInfo["IS_IDENTITY"] is bool isIdentity && isIdentity) { - keyColumns.Add((columnName, keyOrdinal)); + catalogMetadata.IdentityColumns.Add(columnName); } + + // A column the catalog does not describe as non-nullable is treated as nullable, so + // an unreadable flag can only disqualify a unique key, never promote one. + nullabilityByColumn[columnName] = columnInfo["IS_NULLABLE"] is not bool isNullable || isNullable; + + // A column outside every eligible unique index carries nulls for the index members. + if (columnInfo["INDEX_ID"] is not int indexId + || columnInfo["KEY_ORDINAL"] is not byte keyOrdinal) + { + continue; + } + + if (!indexKeysByIndexId.TryGetValue(indexId, out CatalogIndexKey? indexKey)) + { + indexKey = new CatalogIndexKey + { + IsPrimaryKey = columnInfo["IS_PRIMARY_KEY"] is bool isPrimaryKey && isPrimaryKey + }; + indexKeysByIndexId[indexId] = indexKey; + } + + indexKey.Columns.Add((columnName, keyOrdinal)); } - keyColumns.Sort((left, right) => left.KeyOrdinal.CompareTo(right.KeyOrdinal)); + // Index order is creation order, which makes the candidate choice deterministic. + List indexIds = new(indexKeysByIndexId.Keys); + indexIds.Sort(); - foreach ((string ColumnName, byte KeyOrdinal) keyColumn in keyColumns) + foreach (int indexId in indexIds) { - catalogMetadata.PrimaryKeyColumns.Add(keyColumn.ColumnName); + CatalogIndexKey indexKey = indexKeysByIndexId[indexId]; + indexKey.Columns.Sort((left, right) => left.KeyOrdinal.CompareTo(right.KeyOrdinal)); + + List keyColumns = new(); + bool holdsNoNull = true; + + foreach ((string ColumnName, byte KeyOrdinal) keyColumn in indexKey.Columns) + { + keyColumns.Add(keyColumn.ColumnName); + + if (nullabilityByColumn.TryGetValue(keyColumn.ColumnName, out bool isNullable) && isNullable) + { + holdsNoNull = false; + } + } + + if (indexKey.IsPrimaryKey) + { + catalogMetadata.PrimaryKeyColumns.AddRange(keyColumns); + } + else if (holdsNoNull) + { + // Matches the data adapter, which promotes a unique key to the primary key only + // when none of its columns can hold a null. + catalogMetadata.UniqueKeyCandidates.Add(keyColumns); + } } return catalogMetadata; } + /// + /// One unique index of a database object, while its key columns are being collected. + /// + private sealed class CatalogIndexKey + { + public bool IsPrimaryKey { get; init; } + + public List<(string ColumnName, byte KeyOrdinal)> Columns { get; } = new(); + } + public override string GetDefaultSchemaName() { return "dbo"; diff --git a/src/Core/Services/MetadataProviders/SqlMetadataProvider.cs b/src/Core/Services/MetadataProviders/SqlMetadataProvider.cs index 1c731c44ec..872df289eb 100644 --- a/src/Core/Services/MetadataProviders/SqlMetadataProvider.cs +++ b/src/Core/Services/MetadataProviders/SqlMetadataProvider.cs @@ -125,9 +125,13 @@ public abstract class SqlMetadataProvider : private readonly ConcurrentDictionary _objectCatalogMetadataCache = new(StringComparer.Ordinal); /// - /// The two catalog facts an explicit schema projection needs and the "Columns" schema - /// collection does not carry: which columns the database hides from "SELECT *", and which - /// columns form the object's own primary key. + /// The catalog facts an explicit schema projection needs and the "Columns" schema collection + /// does not carry: which columns the database hides from "SELECT *", which columns identify + /// a row, and which are identity columns. Together they replace what the data adapter + /// reports under CommandBehavior.KeyInfo, which cannot be used once the projection is + /// narrowed: the adapter appends key columns the projection left out as hidden reader + /// columns, and an unsupported type among them reintroduces the very failure the narrowing + /// avoids. /// protected sealed class ObjectCatalogMetadata { @@ -140,13 +144,30 @@ protected sealed class ObjectCatalogMetadata public HashSet HiddenColumns { get; } = new(StringComparer.OrdinalIgnoreCase); /// - /// The columns of the object's own primary key, in key order. Replaces the key discovery - /// the data adapter performs under CommandBehavior.KeyInfo, which cannot be used once - /// the projection is narrowed: the adapter appends key columns the projection left out - /// as hidden reader columns, and an unsupported type among them reintroduces the very - /// failure the narrowing avoids. + /// Identity columns. Carried separately rather than through + /// , whose setter coerces a DataType it cannot + /// increment to Int32: SQL Server allows identity on tinyint, numeric and decimal, and + /// the coercion would report the wrong SystemType, which reaches parameter typing and + /// the generated API schemas. + /// + public HashSet IdentityColumns { get; } = new(StringComparer.OrdinalIgnoreCase); + + /// + /// The columns of the object's own primary key, in key order. Empty when the object has + /// none. /// public List PrimaryKeyColumns { get; } = new(); + + /// + /// Unique indexes eligible to identify a row when the object has no primary key, in + /// index order, each holding its key columns in key order. Only indexes whose every key + /// column is non-nullable qualify. + /// Preserves what the data adapter does on the unnarrowed path: absent a primary key it + /// reports such a unique key as , and dropping that + /// would make the narrowed path demand source.key-fields for an object the other path + /// resolves on its own. + /// + public List> UniqueKeyCandidates { get; } = new(); } /// @@ -1686,6 +1707,8 @@ private async Task PopulateSourceDefinitionAsync( sourceDefinition.Columns.TryAdd(columnName, column); } + ApplyIdentityColumnsFromCatalog(schemaName, tableName, sourceDefinition); + RejectPrimaryKeyOnUnsupportedColumn(schemaName, tableName, sourceDefinition); RejectConfiguredReferencesToSkippedColumns(entityName, entity, schemaName, tableName); @@ -1924,11 +1947,20 @@ private async Task FillSchemaForTableAsync( // "Max Pool Size=1" the inner open waits for a connection the outer scope still holds. string projection = await BuildSchemaProjectionAsync(schemaName, tableName); + bool isProjectionNarrowed = !string.Equals(projection, "*", StringComparison.Ordinal); + + // Resolved before the connection below is opened, for the same reason as the projection: + // reading the catalog uses a connection of its own, and nesting that inside an already + // open one exhausts a small pool. + ObjectCatalogMetadata? catalogMetadata = isProjectionNarrowed + ? await GetCachedObjectCatalogMetadataAsync(schemaName, tableName) + : null; + await conn.OpenAsync(); string selectStatement = $"SELECT {projection} FROM {tableNameWithSchemaPrefix}"; - if (!string.Equals(projection, "*", StringComparison.Ordinal)) + if (isProjectionNarrowed) { // The projection left columns out, so the data adapter cannot be used here: // FillSchema runs with CommandBehavior.KeyInfo, under which the provider performs @@ -1944,7 +1976,8 @@ private async Task FillSchemaForTableAsync( selectStatement, tableNameWithSchemaPrefix, schemaName, - tableName); + tableName, + catalogMetadata); } DataAdapterT adapterForTable = new(); @@ -1970,7 +2003,8 @@ private async Task ReadSchemaWithoutKeyInfoAsync( string selectStatement, string tableNameWithSchemaPrefix, string schemaName, - string tableName) + string tableName, + ObjectCatalogMetadata? catalogMetadata) { DataTable dataTable = new(tableNameWithSchemaPrefix); @@ -2001,44 +2035,87 @@ private async Task ReadSchemaWithoutKeyInfoAsync( continue; } + // The provider-reported DataType is carried through unchanged. Auto-increment is + // deliberately not set here — see ApplyIdentityColumnsFromCatalog for why + // DataColumn.AutoIncrement cannot be the transport for it. DataColumn column = new(columnName, (Type)columnInfo["DataType"]) { - AllowDBNull = columnInfo["AllowDBNull"] is bool allowDbNull && allowDbNull, - AutoIncrement = columnInfo["IsAutoIncrement"] is bool isAutoIncrement && isAutoIncrement + // Unknown nullability is treated as nullable. A column wrongly marked + // non-nullable is reported as required through REST, GraphQL and OpenAPI and + // rejects writes the database would accept, so the permissive direction is + // the safe one when the provider does not report the flag. + AllowDBNull = columnInfo["AllowDBNull"] is not bool allowDbNull || allowDbNull }; dataTable.Columns.Add(column); } } - ObjectCatalogMetadata? catalogMetadata = - await GetCachedObjectCatalogMetadataAsync(schemaName, tableName); - - if (catalogMetadata is not null && catalogMetadata.PrimaryKeyColumns.Count > 0) + if (catalogMetadata is not null) { - List keyColumns = new(); + DataColumn[]? keyColumns = ResolveKeyColumns(dataTable, catalogMetadata); - foreach (string primaryKeyColumn in catalogMetadata.PrimaryKeyColumns) + if (keyColumns is not null) { - if (dataTable.Columns.Contains(primaryKeyColumn)) - { - keyColumns.Add(dataTable.Columns[primaryKeyColumn]!); - } + dataTable.PrimaryKey = keyColumns; } + } + + EntitiesDataSet.Tables.Add(dataTable); + + return dataTable; + } + + /// + /// Picks the columns to report as on the narrowed path. + /// The object's own primary key wins. A key column the projection left out is not reported, + /// and the key is not silently replaced by another candidate either: + /// PopulateSourceDefinitionAsync fails through RejectUnreadablePrimaryKey when that key is + /// the one in effect, and a key configured through source.key-fields takes precedence over + /// this one anyway. + /// Absent a primary key, the first unique index whose every key column is present is used, + /// which is what the data adapter reports on the unnarrowed path. Returns null when nothing + /// identifies a row, leaving the caller to report a missing primary key as it does today. + /// + private static DataColumn[]? ResolveKeyColumns(DataTable dataTable, ObjectCatalogMetadata catalogMetadata) + { + if (catalogMetadata.PrimaryKeyColumns.Count > 0) + { + return TryResolveColumns(dataTable, catalogMetadata.PrimaryKeyColumns); + } + + foreach (List uniqueKeyCandidate in catalogMetadata.UniqueKeyCandidates) + { + DataColumn[]? keyColumns = TryResolveColumns(dataTable, uniqueKeyCandidate); - // A key column left out of the projection cannot be reported as a key here, and the - // key is not silently dropped either: PopulateSourceDefinitionAsync fails through - // RejectUnreadablePrimaryKey when that key is the one in effect, while a key - // configured through source.key-fields takes precedence over this one anyway. - if (keyColumns.Count == catalogMetadata.PrimaryKeyColumns.Count) + if (keyColumns is not null) { - dataTable.PrimaryKey = keyColumns.ToArray(); + return keyColumns; } } - EntitiesDataSet.Tables.Add(dataTable); + return null; + } - return dataTable; + /// + /// Resolves every named column against the table, in the order given, or returns null when + /// one of them is absent. + /// + private static DataColumn[]? TryResolveColumns(DataTable dataTable, List columnNames) + { + DataColumn[] columns = new DataColumn[columnNames.Count]; + + for (int index = 0; index < columnNames.Count; index++) + { + if (!dataTable.Columns.Contains(columnNames[index])) + { + return null; + } + + columns[index] = dataTable.Columns[columnNames[index]]!; + } + + return columns; } /// @@ -2065,19 +2142,14 @@ private async Task BuildSchemaProjectionAsync(string schemaName, string try { - ObjectCatalogMetadata? catalogMetadata = - await GetCachedObjectCatalogMetadataAsync(schemaName, tableName); - - if (catalogMetadata is null) - { - // Without the catalog there is no way to tell which columns the database hides - // from "SELECT *". Naming columns anyway would add the hidden period columns of - // a temporal table to the exposed contract, so leave the projection alone. - return "*"; - } - DataTable columnsInTable = await GetCachedColumnsAsync(schemaName, tableName); + // Classify from the "Columns" schema collection first, which is read for every + // object anyway. Only an object that actually holds an unsupported type needs the + // additional catalog facts below; asking for them up front would add a query per + // MSSQL and DWSQL object to every startup. + List supportedColumns = new(); + foreach (DataRow columnInfo in columnsInTable.Rows) { if (columnInfo["COLUMN_NAME"] is not string columnName) @@ -2095,9 +2167,42 @@ private async Task BuildSchemaProjectionAsync(string schemaName, string if (UnsupportedColumnDataTypes.Contains(dataType)) { skippedColumns[columnName] = dataType; - continue; } + else + { + supportedColumns.Add(columnName); + } + } + + if (skippedColumns.Count == 0) + { + return "*"; + } + + if (supportedColumns.Count == 0) + { + // Falling back to "*" here would re-issue the very projection that fails, so the + // caller would see the opaque provider error instead of the reason for it. + throw new DataApiBuilderException( + message: $"Every column of {schemaName}.{tableName} has a data type that is not supported: " + + $"{FormatSkippedColumns(skippedColumns)}. The object cannot be exposed.", + statusCode: HttpStatusCode.ServiceUnavailable, + subStatusCode: DataApiBuilderException.SubStatusCodes.ErrorInInitialization); + } + + ObjectCatalogMetadata? catalogMetadata = + await GetCachedObjectCatalogMetadataAsync(schemaName, tableName); + + if (catalogMetadata is null) + { + // Without the catalog there is no way to tell which columns the database hides + // from "SELECT *". Naming columns anyway would add the hidden period columns of + // a temporal table to the exposed contract, so leave the projection alone. + return "*"; + } + foreach (string columnName in supportedColumns) + { if (catalogMetadata.HiddenColumns.Contains(columnName)) { // "SELECT *" does not return this column, so naming it would widen the @@ -2123,18 +2228,16 @@ private async Task BuildSchemaProjectionAsync(string schemaName, string return "*"; } - if (skippedColumns.Count == 0) - { - return "*"; - } - if (readableColumns.Count == 0) { - // Falling back to "*" here would re-issue the very projection that fails, so the - // caller would see the opaque provider error instead of the reason for it. + // Every column that is not of an unsupported type is one the database hides from + // "SELECT *". Naming the hidden ones is not an option, and "*" would re-issue the + // projection that fails, so nothing about this object can be read. throw new DataApiBuilderException( - message: $"Every column of {schemaName}.{tableName} has a data type that is not supported: " - + $"{FormatSkippedColumns(skippedColumns)}. The object cannot be exposed.", + message: $"No column of {schemaName}.{tableName} can be read: " + + $"{FormatSkippedColumns(skippedColumns)} have a data type that is not supported, and " + + "every remaining column is one the database does not return from a SELECT *. " + + "The object cannot be exposed.", statusCode: HttpStatusCode.ServiceUnavailable, subStatusCode: DataApiBuilderException.SubStatusCodes.ErrorInInitialization); } @@ -2182,6 +2285,41 @@ private void RejectPrimaryKeyOnUnsupportedColumn( } } + /// + /// Marks identity columns on the narrowed schema discovery path, where the shape is read + /// without CommandBehavior.KeyInfo and the reader reports no auto-increment flag. + /// The flag is not carried through : its setter + /// coerces a DataType it cannot increment to Int32, and SQL Server allows identity on + /// tinyint, numeric and decimal, so doing that would report the wrong SystemType and reach + /// parameter typing and the generated API schemas. It comes from the catalog instead. + /// No-op for every object whose projection was not narrowed, which is where the adapter + /// still reports the flag itself. + /// + private void ApplyIdentityColumnsFromCatalog( + string schemaName, + string tableName, + SourceDefinition sourceDefinition) + { + if (!_objectCatalogMetadataCache.TryGetValue( + GetObjectCacheKey(schemaName, tableName), + out ObjectCatalogMetadata? catalogMetadata)) + { + return; + } + + foreach (string identityColumn in catalogMetadata.IdentityColumns) + { + if (sourceDefinition.Columns.TryGetValue(identityColumn, out ColumnDefinition? columnDefinition)) + { + columnDefinition.IsAutoGenerated = true; + + // Matches how the adapter-reported flag is treated above: an auto-increment + // column is also read-only. + columnDefinition.IsReadOnly = true; + } + } + } + /// /// Fails initialization with the reason when the object's own primary key includes a column /// left out of the projection because its data type is not supported. Without this the diff --git a/src/Service.Tests/DatabaseSchema-MsSql.sql b/src/Service.Tests/DatabaseSchema-MsSql.sql index d94ef77c48..01ae49df12 100644 --- a/src/Service.Tests/DatabaseSchema-MsSql.sql +++ b/src/Service.Tests/DatabaseSchema-MsSql.sql @@ -49,6 +49,7 @@ DROP TABLE IF EXISTS geometry_type_table; DROP TABLE IF EXISTS hierarchyid_pk_table; DROP TABLE IF EXISTS hierarchyid_composite_pk_table; DROP TABLE IF EXISTS hierarchyid_unique_table; +DROP TABLE IF EXISTS unique_key_geometry_table; -- System versioning has to be released before the temporal table can be dropped. IF OBJECT_ID('dbo.temporal_geometry_type_table', 'U') IS NOT NULL AND OBJECTPROPERTY(OBJECT_ID('dbo.temporal_geometry_type_table'), 'TableTemporalType') = 2 @@ -306,6 +307,16 @@ CREATE TABLE hierarchyid_unique_table( CONSTRAINT UQ_hierarchyid_unique_table_node UNIQUE (node) ); +-- No database primary key, and the unique index covers a supported non-null column. The data +-- adapter promotes such a key to DataTable.PrimaryKey on the unnarrowed path, so the narrowed path +-- has to infer it too instead of demanding source.key-fields. +CREATE TABLE unique_key_geometry_table( + code varchar(20) NOT NULL, + name varchar(100) NOT NULL, + geom geometry NULL, + CONSTRAINT UQ_unique_key_geometry_table_code UNIQUE (code) +); + CREATE TABLE profiles( id int IDENTITY(5001, 1) PRIMARY KEY, metadata json NULL @@ -731,6 +742,11 @@ VALUES (1, hierarchyid::Parse('/1/'), 'root'); INSERT INTO hierarchyid_unique_table(id, node, name) VALUES (1, hierarchyid::Parse('/1/'), 'root'); +INSERT INTO unique_key_geometry_table(code, name, geom) +VALUES + ('CAR-001', 'point', geometry::STGeomFromText('POINT(1 2)', 0)), + ('CAR-002', 'null geometry', NULL); + SET IDENTITY_INSERT profiles ON INSERT INTO profiles(id, metadata) VALUES diff --git a/src/Service.Tests/UnitTests/SqlMetadataProviderUnitTests.cs b/src/Service.Tests/UnitTests/SqlMetadataProviderUnitTests.cs index 68f56a80c2..9313551fb8 100644 --- a/src/Service.Tests/UnitTests/SqlMetadataProviderUnitTests.cs +++ b/src/Service.Tests/UnitTests/SqlMetadataProviderUnitTests.cs @@ -750,6 +750,49 @@ await SetUpSingleEntityMetadataProviderAsync( TestHelper.UnsetAllDABEnvironmentVariables(); } + /// + /// Test to validate that an object with no database primary key, whose unique index covers a + /// supported non-null column, has that key inferred without `source.key-fields`. + /// `DbDataAdapter.FillSchema` promotes such a unique key to `DataTable.PrimaryKey` on the + /// unnarrowed path, so dropping it once the projection is narrowed would make the two paths + /// disagree and demand configuration for an object the other path resolves on its own. + /// + [TestMethod, TestCategory(TestCategory.MSSQL)] + public async Task ValidateUniqueKeyIsInferredWhenNoDatabasePrimaryKeyExists() + { + DatabaseEngine = TestCategory.MSSQL; + TestHelper.SetupDatabaseEnvironment(DatabaseEngine); + + await SetUpSingleEntityMetadataProviderAsync( + "UniqueKeyGeometry", + BuildReadOnlyEntity( + entityName: "UniqueKeyGeometry", + databaseObject: "dbo.unique_key_geometry_table", + sourceType: EntitySourceType.Table, + keyFields: null)); + + await _sqlMetadataProvider.InitializeAsync(); + + Assert.IsTrue( + _sqlMetadataProvider.GetEntityNamesAndDbObjects().TryGetValue("UniqueKeyGeometry", out DatabaseObject databaseObject), + message: "Metadata inference failed for an object whose only key is a unique index over a supported column."); + + SourceDefinition sourceDefinition = databaseObject.SourceDefinition; + + CollectionAssert.AreEqual( + new List { "code" }, + sourceDefinition.PrimaryKey, + message: "The non-null unique column is expected to be inferred as the primary key."); + Assert.IsTrue( + sourceDefinition.Columns.ContainsKey("name"), + message: "A column with a supported data type is expected in the source definition."); + Assert.IsFalse( + sourceDefinition.Columns.ContainsKey("geom"), + message: "A column whose data type cannot be mapped is not expected in the source definition."); + + TestHelper.UnsetAllDABEnvironmentVariables(); + } + /// /// Builds a metadata provider over a single in-memory entity and resets the database state. /// The objects exercised by the unsupported-data-type tests are declared in memory rather From f4eacf61d31ba1e76a38b9b0e8dec53096ccde16 Mon Sep 17 00:00:00 2001 From: Joymax Andrade do Nascimento Date: Mon, 14 Sep 2026 22:57:50 -0300 Subject: [PATCH 12/19] Reject database policies over skipped columns and cover identity metadata A database policy is parsed per request against the OData model, which EdmModelBuilder builds from SourceDefinition.Columns, so one naming a column left out of the projection returns 400 on every request for that role. Configured references in mappings, fields, fields.include and policy.database are now all rejected at initialization. Policy field references are scanned rather than parsed, because the model the parser needs does not exist yet at that point. Adds identity assertions to ValidateUnsupportedColumnTypeIsNotInferred, plus decimal_identity_geometry_table and ValidateIdentityTypeIsPreservedOnTheNarrowedPath, which is what verifies that carrying identity from the catalog preserves a DataType that DataColumn.AutoIncrement would coerce to Int32. Addresses the second Copilot review on #3802. --- .../MetadataProviders/SqlMetadataProvider.cs | 50 ++++++++ src/Service.Tests/DatabaseSchema-MsSql.sql | 14 +++ .../UnitTests/SqlMetadataProviderUnitTests.cs | 107 ++++++++++++++++++ 3 files changed, 171 insertions(+) diff --git a/src/Core/Services/MetadataProviders/SqlMetadataProvider.cs b/src/Core/Services/MetadataProviders/SqlMetadataProvider.cs index 872df289eb..85891fc7bf 100644 --- a/src/Core/Services/MetadataProviders/SqlMetadataProvider.cs +++ b/src/Core/Services/MetadataProviders/SqlMetadataProvider.cs @@ -2400,6 +2400,15 @@ private void RejectConfiguredReferencesToSkippedColumns( { foreach (EntityAction action in permission.Actions) { + // A database policy is parsed per request against the OData model, which is + // built from SourceDefinition.Columns. A policy naming a column that is not + // there fails every request for that role instead of the configuration being + // rejected once, at startup. + foreach (string policyField in EnumeratePolicyFieldReferences(action.Policy?.Database)) + { + RejectReference(policyField, $"database policy of role {permission.Role}"); + } + if (action.Fields?.Include is null) { continue; @@ -2431,6 +2440,47 @@ void RejectReference(string configuredName, string configurationSection) } } + /// + /// Yields the field names a database policy references through its "@item." prefix. + /// Scanned by hand rather than parsed: the OData model the parser needs does not exist yet + /// at this point in initialization, and a reference this scan fails to recognize only means + /// one fewer configuration rejected at startup — never a wrong rejection. + /// A policy can only reach a skipped column by its backing name, because an alias over one + /// is rejected through mappings and fields before this runs. + /// + private static IEnumerable EnumeratePolicyFieldReferences(string? databasePolicy) + { + const string POLICY_FIELD_PREFIX = "@item."; + + if (string.IsNullOrWhiteSpace(databasePolicy)) + { + yield break; + } + + int prefixIndex = databasePolicy.IndexOf(POLICY_FIELD_PREFIX, StringComparison.OrdinalIgnoreCase); + + while (prefixIndex >= 0) + { + int fieldStart = prefixIndex + POLICY_FIELD_PREFIX.Length; + int fieldEnd = fieldStart; + + while (fieldEnd < databasePolicy.Length + && (char.IsLetterOrDigit(databasePolicy[fieldEnd]) || databasePolicy[fieldEnd] == '_')) + { + fieldEnd++; + } + + if (fieldEnd > fieldStart) + { + yield return databasePolicy[fieldStart..fieldEnd]; + } + + prefixIndex = fieldEnd >= databasePolicy.Length + ? -1 + : databasePolicy.IndexOf(POLICY_FIELD_PREFIX, fieldEnd, StringComparison.OrdinalIgnoreCase); + } + } + /// /// Renders skipped columns as "name (type)" pairs for log and error messages. /// diff --git a/src/Service.Tests/DatabaseSchema-MsSql.sql b/src/Service.Tests/DatabaseSchema-MsSql.sql index 01ae49df12..1d2a8be83f 100644 --- a/src/Service.Tests/DatabaseSchema-MsSql.sql +++ b/src/Service.Tests/DatabaseSchema-MsSql.sql @@ -50,6 +50,7 @@ DROP TABLE IF EXISTS hierarchyid_pk_table; DROP TABLE IF EXISTS hierarchyid_composite_pk_table; DROP TABLE IF EXISTS hierarchyid_unique_table; DROP TABLE IF EXISTS unique_key_geometry_table; +DROP TABLE IF EXISTS decimal_identity_geometry_table; -- System versioning has to be released before the temporal table can be dropped. IF OBJECT_ID('dbo.temporal_geometry_type_table', 'U') IS NOT NULL AND OBJECTPROPERTY(OBJECT_ID('dbo.temporal_geometry_type_table'), 'TableTemporalType') = 2 @@ -317,6 +318,14 @@ CREATE TABLE unique_key_geometry_table( CONSTRAINT UQ_unique_key_geometry_table_code UNIQUE (code) ); +-- A decimal identity column. DataColumn.AutoIncrement would coerce its CLR type to Int32, which is +-- why identity is carried from the catalog instead; this fixture is what proves the type survives. +CREATE TABLE decimal_identity_geometry_table( + id decimal(18, 0) IDENTITY(1, 1) NOT NULL PRIMARY KEY, + name varchar(100) NOT NULL, + geom geometry NULL +); + CREATE TABLE profiles( id int IDENTITY(5001, 1) PRIMARY KEY, metadata json NULL @@ -747,6 +756,11 @@ VALUES ('CAR-001', 'point', geometry::STGeomFromText('POINT(1 2)', 0)), ('CAR-002', 'null geometry', NULL); +INSERT INTO decimal_identity_geometry_table(name, geom) +VALUES + ('point', geometry::STGeomFromText('POINT(1 2)', 0)), + ('null geometry', NULL); + SET IDENTITY_INSERT profiles ON INSERT INTO profiles(id, metadata) VALUES diff --git a/src/Service.Tests/UnitTests/SqlMetadataProviderUnitTests.cs b/src/Service.Tests/UnitTests/SqlMetadataProviderUnitTests.cs index 9313551fb8..cc79832e0f 100644 --- a/src/Service.Tests/UnitTests/SqlMetadataProviderUnitTests.cs +++ b/src/Service.Tests/UnitTests/SqlMetadataProviderUnitTests.cs @@ -426,6 +426,113 @@ public async Task ValidateUnsupportedColumnTypeIsNotInferred() sourceDefinition.Columns.ContainsKey("geom"), message: "A column whose data type cannot be mapped is not expected in the source definition."); + // The narrowed path reads the shape without CommandBehavior.KeyInfo, so the reader + // reports no auto-increment flag and it is carried from the catalog instead. Asserted + // here because losing it would silently make creates require an identity value. + Assert.IsTrue( + sourceDefinition.Columns["id"].IsAutoGenerated, + message: "The identity column is expected to be marked auto-generated."); + Assert.IsTrue( + sourceDefinition.Columns["id"].IsReadOnly, + message: "An auto-generated column is expected to be read-only."); + + TestHelper.UnsetAllDABEnvironmentVariables(); + } + + /// + /// Test to validate that an identity column whose CLR type cannot be auto-incremented keeps + /// the type the provider reported. + /// `DataColumn.AutoIncrement` coerces such a DataType to Int32, and SQL Server allows + /// identity on tinyint, numeric and decimal, so carrying the flag through that property + /// would report the wrong SystemType and reach parameter typing and the generated API + /// schemas. This is the fixture that proves the type survives. + /// + [TestMethod, TestCategory(TestCategory.MSSQL)] + public async Task ValidateIdentityTypeIsPreservedOnTheNarrowedPath() + { + DatabaseEngine = TestCategory.MSSQL; + TestHelper.SetupDatabaseEnvironment(DatabaseEngine); + + await SetUpSingleEntityMetadataProviderAsync( + "DecimalIdentityGeometry", + BuildReadOnlyEntity( + entityName: "DecimalIdentityGeometry", + databaseObject: "dbo.decimal_identity_geometry_table", + sourceType: EntitySourceType.Table, + keyFields: null)); + + await _sqlMetadataProvider.InitializeAsync(); + + Assert.IsTrue( + _sqlMetadataProvider.GetEntityNamesAndDbObjects().TryGetValue("DecimalIdentityGeometry", out DatabaseObject databaseObject), + message: "Metadata inference failed for an object with a decimal identity column."); + + SourceDefinition sourceDefinition = databaseObject.SourceDefinition; + + Assert.AreEqual( + typeof(decimal), + sourceDefinition.Columns["id"].SystemType, + message: "The identity column is expected to keep the data type the provider reported."); + Assert.IsTrue( + sourceDefinition.Columns["id"].IsAutoGenerated, + message: "The identity column is expected to be marked auto-generated."); + Assert.IsTrue( + sourceDefinition.Columns["id"].IsReadOnly, + message: "An auto-generated column is expected to be read-only."); + Assert.IsFalse( + sourceDefinition.Columns.ContainsKey("geom"), + message: "A column whose data type cannot be mapped is not expected in the source definition."); + + TestHelper.UnsetAllDABEnvironmentVariables(); + } + + /// + /// Test to validate that a database policy referencing a column left out of the projection + /// fails initialization. + /// A policy is parsed per request against the OData model, which is built from + /// `SourceDefinition.Columns`, so a policy naming an absent column returns 400 on every + /// request for that role instead of the configuration being rejected once, at startup. + /// + [TestMethod, TestCategory(TestCategory.MSSQL)] + public async Task ValidateDatabasePolicyOverUnsupportedColumnFailsInitialization() + { + DatabaseEngine = TestCategory.MSSQL; + TestHelper.SetupDatabaseEnvironment(DatabaseEngine); + + Entity entity = new( + Source: new("dbo.geometry_type_table", EntitySourceType.Table, null, new string[] { "id" }), + Fields: null, + Rest: new(Enabled: true), + GraphQL: new("GeometryPolicy", "GeometryPolicys", Enabled: true), + Permissions: new EntityPermission[] + { + new(Role: "anonymous", + Actions: new EntityAction[] + { + new(Action: EntityActionOperation.Read, + Fields: null, + Policy: new(Request: null, Database: "@item.geom eq null")) + }) + }, + Relationships: null, + Mappings: null); + + await SetUpSingleEntityMetadataProviderAsync("GeometryPolicy", entity); + + try + { + await _sqlMetadataProvider.InitializeAsync(); + Assert.Fail("Expected DataApiBuilderException was not thrown for a database policy over a column of an unsupported data type."); + } + catch (DataApiBuilderException ex) + { + Assert.AreEqual(HttpStatusCode.ServiceUnavailable, ex.StatusCode); + Assert.AreEqual(DataApiBuilderException.SubStatusCodes.ErrorInInitialization, ex.SubStatusCode); + Assert.IsTrue( + ex.Message.Contains("geom") && ex.Message.Contains("policy"), + message: $"The error is expected to name the column and the policy referencing it. Actual message: {ex.Message}"); + } + TestHelper.UnsetAllDABEnvironmentVariables(); } From 7584b5275b86a6449ce3ed259657aa0153fe8cb0 Mon Sep 17 00:00:00 2001 From: Joymax Andrade do Nascimento Date: Mon, 21 Sep 2026 20:21:35 -0300 Subject: [PATCH 13/19] Infer view keys from the base object catalog A view has no index of its own, so the key was taken from the column-level is_part_of_unique_key flag. That flag is a union across every unique key of the underlying object and cannot be split back into one key, so a projection that exposed only part of a key could have that part inferred as the key. A partial key silently matches more than one row on an update or a delete. Resolve the projection's source object through browse mode, read that object's real primary key and unique candidates from the catalog, and accept only a key the projection exposes in full. When no key qualifies, fail naming the column that is missing rather than reporting a generic missing primary key. Add the TemporalGeometryType entity with a PUT test showing that hidden period columns are neither inferred into the source definition nor nullified by AddNullifiedUnspecifiedFields, plus MSSQL fixtures and tests for hierarchyid keys, tables whose only key is a unique index, and views over columns the provider cannot map. --- config-generators/mssql-commands.txt | 1 + .../MsSqlMetadataProvider.cs | 211 ++++++++++- .../MetadataProviders/SqlMetadataProvider.cs | 160 +++++++-- src/Service.Tests/DatabaseSchema-MsSql.sql | 28 ++ ...tReadingRuntimeConfigForMsSql.verified.txt | 32 ++ .../RestApiTests/Put/MsSqlPutApiTests.cs | 36 +- .../UnitTests/SqlMetadataProviderUnitTests.cs | 336 +++++++++++++++++- src/Service.Tests/dab-config.MsSql.json | 32 ++ 8 files changed, 811 insertions(+), 25 deletions(-) diff --git a/config-generators/mssql-commands.txt b/config-generators/mssql-commands.txt index 53674ef59c..407b4167b1 100644 --- a/config-generators/mssql-commands.txt +++ b/config-generators/mssql-commands.txt @@ -23,6 +23,7 @@ update VectorType --config "dab-config.MsSql.json" --permissions "authenticated: update VectorOwner --config "dab-config.MsSql.json" --relationship vectors --target.entity VectorType --cardinality many --relationship.fields "id:owner_id" update VectorType --config "dab-config.MsSql.json" --relationship owner --target.entity VectorOwner --cardinality one --relationship.fields "owner_id:id" add GeometryType --config "dab-config.MsSql.json" --source geometry_type_table --rest true --graphql true --permissions "anonymous:read" +add TemporalGeometryType --config "dab-config.MsSql.json" --source temporal_geometry_type_table --rest true --graphql true --permissions "anonymous:create,read,update" add Profile --config "dab-config.MsSql.json" --source profiles --rest true --graphql true --permissions "anonymous:create,read,delete,update" update Profile --config "dab-config.MsSql.json" --permissions "authenticated:create,read,delete,update" add stocks_price --config "dab-config.MsSql.json" --source stocks_price --permissions "authenticated:create,read,update,delete" diff --git a/src/Core/Services/MetadataProviders/MsSqlMetadataProvider.cs b/src/Core/Services/MetadataProviders/MsSqlMetadataProvider.cs index cde419fd6e..39885ee75a 100644 --- a/src/Core/Services/MetadataProviders/MsSqlMetadataProvider.cs +++ b/src/Core/Services/MetadataProviders/MsSqlMetadataProvider.cs @@ -115,7 +115,10 @@ public MsSqlMetadataProvider( // sys.columns.is_hidden exists from SQL Server 2016 on. Where the catalog cannot // answer — a dedicated SQL pool, or a login without VIEW DEFINITION — returning null // leaves the projection at "*", which is exactly the behavior before this change. - _logger.LogDebug( + // Logged at Warning, not Debug: the fallback changes what the object exposes — the + // projection stays "SELECT *" and an unsupported column takes the entity down with + // the provider's own opaque error — so the reason has to be visible by default. + _logger.LogWarning( "Unable to read catalog metadata for {schemaName}.{tableName}: {message}", schemaName, tableName, @@ -225,6 +228,212 @@ public MsSqlMetadataProvider( return catalogMetadata; } + /// + protected override async Task> GetProjectionKeyFromResultSetAsync(string selectStatement) + { + List projectionColumns = await DescribeProjectionAsync(selectStatement); + + if (projectionColumns.Count == 0) + { + return new List(); + } + + // The key is taken from the underlying object's own catalog entry rather than from + // is_part_of_unique_key, which is reported per column as the union of every unique key + // of the result. That union cannot be split back into one key: a view over a table with + // both a primary key and a unique index would yield the two fused into an + // over-specified key, and a key member the projection left out comes back flagged + // hidden, which would yield a partial key that does not identify a row at all. + string? sourceSchema = null; + string? sourceTable = null; + Dictionary projectionBySourceColumn = new(StringComparer.OrdinalIgnoreCase); + + foreach (ProjectionSourceColumn projectionColumn in projectionColumns) + { + if (projectionColumn.IsHidden + || projectionColumn.SourceSchema is null + || projectionColumn.SourceTable is null + || projectionColumn.SourceColumn is null) + { + continue; + } + + if (sourceSchema is null) + { + sourceSchema = projectionColumn.SourceSchema; + sourceTable = projectionColumn.SourceTable; + } + else if (!string.Equals(sourceSchema, projectionColumn.SourceSchema, StringComparison.Ordinal) + || !string.Equals(sourceTable, projectionColumn.SourceTable, StringComparison.Ordinal)) + { + // More than one underlying object. Inferring a key across a join is out of + // scope; such an entity needs source.key-fields, as it did before this change. + return new List(); + } + + projectionBySourceColumn[projectionColumn.SourceColumn] = projectionColumn.Name; + } + + if (sourceSchema is null || sourceTable is null) + { + return new List(); + } + + ObjectCatalogMetadata? sourceCatalogMetadata = + await GetCachedObjectCatalogMetadataAsync(sourceSchema, sourceTable); + + if (sourceCatalogMetadata is null) + { + return new List(); + } + + // The underlying object's primary key comes first, then its unique keys, in index order + // — the preference the table path applies. Unlike a table, though, a view can simply + // not select the primary key, which is ordinary design rather than a key it cannot + // express, so a primary key that is out of reach does not end the search: a unique key + // the projection does carry whole identifies a row just as well. + List> sourceKeys = new(); + + if (sourceCatalogMetadata.PrimaryKeyColumns.Count > 0) + { + sourceKeys.Add(sourceCatalogMetadata.PrimaryKeyColumns); + } + + sourceKeys.AddRange(sourceCatalogMetadata.UniqueKeyCandidates); + + string? unreachableKeyColumn = null; + + foreach (List sourceKey in sourceKeys) + { + List projectionKey = new(); + string? missingKeyColumn = null; + + foreach (string sourceKeyColumn in sourceKey) + { + if (!projectionBySourceColumn.TryGetValue(sourceKeyColumn, out string? projectionColumnName)) + { + missingKeyColumn = sourceKeyColumn; + break; + } + + projectionKey.Add(projectionColumnName); + } + + if (missingKeyColumn is null && projectionKey.Count > 0) + { + return projectionKey; + } + + // Remembered from the first key that came closest, to name something concrete if no + // key resolves at all. A partial key is never returned: it silently matches more + // than one row on an update or a delete. + unreachableKeyColumn ??= missingKeyColumn; + } + + if (unreachableKeyColumn is not null) + { + throw new DataApiBuilderException( + message: $"No key of {sourceSchema}.{sourceTable}, which this object is built on, is fully exposed " + + $"by it: the nearest one includes the column {unreachableKeyColumn}, which this object does " + + "not expose. It therefore cannot be reached by key. Configure source.key-fields with columns " + + "it does expose that identify a row uniquely, if there are any.", + statusCode: HttpStatusCode.ServiceUnavailable, + subStatusCode: DataApiBuilderException.SubStatusCodes.ErrorInInitialization); + } + + return new List(); + } + + /// + /// Describes a projection and returns what each of its columns resolves to in the catalog. + /// Browse information is what populates the source_* columns, and it is also what resolves a + /// view's columns to the object underneath. Metadata only: no row is read and no CLR type is + /// requested, so an unsupported column elsewhere in the object cannot fail this. + /// + private async Task> DescribeProjectionAsync(string selectStatement) + { + string statementParamName = $"{BaseQueryStructure.PARAM_NAME_PREFIX}param0"; + + string query = + "select r.name as COLUMN_NAME, r.source_schema as SOURCE_SCHEMA, " + + "r.source_table as SOURCE_TABLE, r.source_column as SOURCE_COLUMN, " + + "r.is_hidden as IS_HIDDEN " + + $"from sys.dm_exec_describe_first_result_set({statementParamName}, null, 1) as r " + + "order by r.column_ordinal;"; + + Dictionary parameters = new() + { + { statementParamName, new(selectStatement, DbType.String) } + }; + + try + { + List? projectionColumns = await QueryExecutor.ExecuteQueryAsync( + sqltext: query, + parameters: parameters, + dataReaderHandler: SummarizeProjectionSourceColumnsAsync, + dataSourceName: _dataSourceName); + + return projectionColumns ?? new List(); + } + catch (Exception ex) + { + // The statement may not be describable, or the dynamic management function may be + // unavailable, as on a dedicated SQL pool. Reporting nothing leaves the behavior to + // the missing-primary-key path, which is where it was before this change. + _logger.LogWarning( + "Unable to describe the projection of {selectStatement} to infer a key: {message}", + selectStatement, + ex.Message); + + return new List(); + } + } + + private async Task> SummarizeProjectionSourceColumnsAsync( + DbDataReader reader, + List? args = null) + { + DbResultSet resultSet = await QueryExecutor.ExtractResultSetFromDbDataReaderAsync(reader); + + List projectionColumns = new(); + + foreach (DbResultSetRow row in resultSet.Rows) + { + if (row.Columns["COLUMN_NAME"] is not string columnName) + { + continue; + } + + projectionColumns.Add(new ProjectionSourceColumn + { + Name = columnName, + SourceSchema = row.Columns["SOURCE_SCHEMA"] as string, + SourceTable = row.Columns["SOURCE_TABLE"] as string, + SourceColumn = row.Columns["SOURCE_COLUMN"] as string, + IsHidden = row.Columns["IS_HIDDEN"] is bool isHidden && isHidden + }); + } + + return projectionColumns; + } + + /// + /// One column of a described projection, with the catalog object it resolves to. + /// + private sealed class ProjectionSourceColumn + { + public string Name { get; init; } = string.Empty; + + public string? SourceSchema { get; init; } + + public string? SourceTable { get; init; } + + public string? SourceColumn { get; init; } + + public bool IsHidden { get; init; } + } + /// /// One unique index of a database object, while its key columns are being collected. /// diff --git a/src/Core/Services/MetadataProviders/SqlMetadataProvider.cs b/src/Core/Services/MetadataProviders/SqlMetadataProvider.cs index 85891fc7bf..fd2b92cdd7 100644 --- a/src/Core/Services/MetadataProviders/SqlMetadataProvider.cs +++ b/src/Core/Services/MetadataProviders/SqlMetadataProvider.cs @@ -183,11 +183,30 @@ protected sealed class ObjectCatalogMetadata return Task.FromResult(null); } + /// + /// Returns the columns of a unique key the given projection exposes, in key order, for an + /// object the catalog holds no index for — a view, whose key the data adapter resolved + /// through the underlying table. Empty by default, and empty whenever the provider cannot + /// answer, which leaves the missing-primary-key error to be reported as before. + /// A key column the projection left out must never be reported as part of a key: a partial + /// key silently matches more than one row on an update or a delete. + /// + /// + /// An implementation may fail here instead of returning empty, when it can tell that no key + /// of the underlying object is fully exposed and can name what is missing. That is a better + /// error than the generic missing-primary-key one, which reads as something the user forgot + /// to configure. + /// + protected virtual Task> GetProjectionKeyFromResultSetAsync(string selectStatement) + { + return Task.FromResult(new List()); + } + /// /// Returns for a database object, reading the catalog /// once per object for the duration of metadata initialization. /// - private async Task GetCachedObjectCatalogMetadataAsync( + protected async Task GetCachedObjectCatalogMetadataAsync( string schemaName, string tableName) { @@ -1956,10 +1975,22 @@ private async Task FillSchemaForTableAsync( ? await GetCachedObjectCatalogMetadataAsync(schemaName, tableName) : null; - await conn.OpenAsync(); - string selectStatement = $"SELECT {projection} FROM {tableNameWithSchemaPrefix}"; + // An ordinary view has no indexes of its own, so the catalog lookup above finds no key + // for it. FillSchema resolved one through the view's underlying table under KeyInfo, and + // describing the projection recovers the same route — without a reader, and without + // asking for a CLR type. Only reached for an object the catalog could not key, and + // resolved before the connection is opened for the same pooling reason as above. + // Kept out of the cached metadata: it depends on this projection, not on the object. + List describedKey = catalogMetadata is not null + && catalogMetadata.PrimaryKeyColumns.Count == 0 + && catalogMetadata.UniqueKeyCandidates.Count == 0 + ? await GetProjectionKeyFromResultSetAsync(selectStatement) + : new List(); + + await conn.OpenAsync(); + if (isProjectionNarrowed) { // The projection left columns out, so the data adapter cannot be used here: @@ -1977,7 +2008,8 @@ private async Task FillSchemaForTableAsync( tableNameWithSchemaPrefix, schemaName, tableName, - catalogMetadata); + catalogMetadata, + describedKey); } DataAdapterT adapterForTable = new(); @@ -2004,7 +2036,8 @@ private async Task ReadSchemaWithoutKeyInfoAsync( string tableNameWithSchemaPrefix, string schemaName, string tableName, - ObjectCatalogMetadata? catalogMetadata) + ObjectCatalogMetadata? catalogMetadata, + List describedKey) { DataTable dataTable = new(tableNameWithSchemaPrefix); @@ -2053,7 +2086,8 @@ private async Task ReadSchemaWithoutKeyInfoAsync( if (catalogMetadata is not null) { - DataColumn[]? keyColumns = ResolveKeyColumns(dataTable, catalogMetadata); + DataColumn[]? keyColumns = ResolveKeyColumns(dataTable, catalogMetadata) + ?? (describedKey.Count > 0 ? TryResolveColumns(dataTable, describedKey) : null); if (keyColumns is not null) { @@ -2286,8 +2320,8 @@ private void RejectPrimaryKeyOnUnsupportedColumn( } /// - /// Marks identity columns on the narrowed schema discovery path, where the shape is read - /// without CommandBehavior.KeyInfo and the reader reports no auto-increment flag. + /// Marks identity columns on the narrowed schema discovery path, which builds its columns + /// itself instead of letting the data adapter populate them. /// The flag is not carried through : its setter /// coerces a DataType it cannot increment to Int32, and SQL Server allows identity on /// tinyint, numeric and decimal, so doing that would report the wrong SystemType and reach @@ -2374,11 +2408,15 @@ private void RejectConfiguredReferencesToSkippedColumns( if (entity is null || !_skippedColumnsByObject.TryGetValue( GetObjectCacheKey(schemaName, tableName), - out Dictionary? skippedColumns)) + out Dictionary? skippedColumnsForObject)) { return; } + // Held in a non-nullable local because the local function below captures it, and the + // guard above is not something the compiler can carry into that capture. + Dictionary skippedColumns = skippedColumnsForObject; + // "mappings" and "fields" both key on the backing column name. if (entity.Mappings is not null) { @@ -2406,7 +2444,16 @@ private void RejectConfiguredReferencesToSkippedColumns( // rejected once, at startup. foreach (string policyField in EnumeratePolicyFieldReferences(action.Policy?.Database)) { - RejectReference(policyField, $"database policy of role {permission.Role}"); + // Policy identifiers are exposed names, the way the OData model's properties + // are, so they are resolved through the configured aliases before being + // compared against backing column names. Without that, an alias over a + // supported column that happens to carry the name of a skipped one — a + // "Location" alias of a text column beside a skipped "Location" spatial + // column — would be rejected even though the policy references the + // supported field. + RejectReference( + ResolveBackingColumnName(entity, policyField), + $"database policy of role {permission.Role}"); } if (action.Fields?.Include is null) @@ -2442,11 +2489,11 @@ void RejectReference(string configuredName, string configurationSection) /// /// Yields the field names a database policy references through its "@item." prefix. + /// Single-quoted literals are skipped, with a doubled quote read as an escaped quote inside + /// one, so a literal that merely contains the prefix is not mistaken for a reference. /// Scanned by hand rather than parsed: the OData model the parser needs does not exist yet - /// at this point in initialization, and a reference this scan fails to recognize only means - /// one fewer configuration rejected at startup — never a wrong rejection. - /// A policy can only reach a skipped column by its backing name, because an alias over one - /// is rejected through mappings and fields before this runs. + /// at this point in initialization. The names yielded are exposed names, which the caller + /// resolves through the configured aliases. /// private static IEnumerable EnumeratePolicyFieldReferences(string? databasePolicy) { @@ -2457,11 +2504,49 @@ private static IEnumerable EnumeratePolicyFieldReferences(string? databa yield break; } - int prefixIndex = databasePolicy.IndexOf(POLICY_FIELD_PREFIX, StringComparison.OrdinalIgnoreCase); + int index = 0; - while (prefixIndex >= 0) + while (index < databasePolicy.Length) { - int fieldStart = prefixIndex + POLICY_FIELD_PREFIX.Length; + if (databasePolicy[index] == '\'') + { + index++; + + while (index < databasePolicy.Length) + { + if (databasePolicy[index] != '\'') + { + index++; + continue; + } + + // A doubled quote is an escaped quote within the literal, not its end. + if (index + 1 < databasePolicy.Length && databasePolicy[index + 1] == '\'') + { + index += 2; + continue; + } + + index++; + break; + } + + continue; + } + + if (string.Compare( + databasePolicy, + index, + POLICY_FIELD_PREFIX, + 0, + POLICY_FIELD_PREFIX.Length, + StringComparison.OrdinalIgnoreCase) != 0) + { + index++; + continue; + } + + int fieldStart = index + POLICY_FIELD_PREFIX.Length; int fieldEnd = fieldStart; while (fieldEnd < databasePolicy.Length @@ -2475,10 +2560,45 @@ private static IEnumerable EnumeratePolicyFieldReferences(string? databa yield return databasePolicy[fieldStart..fieldEnd]; } - prefixIndex = fieldEnd >= databasePolicy.Length - ? -1 - : databasePolicy.IndexOf(POLICY_FIELD_PREFIX, fieldEnd, StringComparison.OrdinalIgnoreCase); + index = fieldEnd > fieldStart ? fieldEnd : fieldStart; + } + } + + /// + /// Resolves an exposed field name to the column it is backed by, using the aliases the + /// configuration declares through "mappings" and through a field's "alias". An unaliased + /// name is already the backing name. + /// + private static string ResolveBackingColumnName(Entity entity, string exposedName) + { + // "fields" is consulted before "mappings", mirroring the precedence + // GenerateExposedToBackingColumnMapUtil applies when it builds the map the runtime + // resolves names through. Reversing it here would reject a configuration the runtime + // resolves the other way. + if (entity.Fields is not null) + { + foreach (FieldMetadata field in entity.Fields) + { + if (!string.IsNullOrWhiteSpace(field.Alias) + && string.Equals(field.Alias, exposedName, StringComparison.OrdinalIgnoreCase)) + { + return field.Name; + } + } + } + + if (entity.Mappings is not null) + { + foreach (KeyValuePair mapping in entity.Mappings) + { + if (string.Equals(mapping.Value, exposedName, StringComparison.OrdinalIgnoreCase)) + { + return mapping.Key; + } + } } + + return exposedName; } /// diff --git a/src/Service.Tests/DatabaseSchema-MsSql.sql b/src/Service.Tests/DatabaseSchema-MsSql.sql index 1d2a8be83f..65a668dbbb 100644 --- a/src/Service.Tests/DatabaseSchema-MsSql.sql +++ b/src/Service.Tests/DatabaseSchema-MsSql.sql @@ -12,6 +12,9 @@ DROP VIEW IF EXISTS stocks_view_selected; DROP VIEW IF EXISTS books_publishers_view_composite; DROP VIEW IF EXISTS books_publishers_view_composite_insertable; DROP VIEW IF EXISTS geometry_only_view; +DROP VIEW IF EXISTS geometry_all_view; +DROP VIEW IF EXISTS hierarchyid_composite_view; +DROP VIEW IF EXISTS unique_without_pk_view; DROP PROCEDURE IF EXISTS get_books; DROP PROCEDURE IF EXISTS get_book_by_id; DROP PROCEDURE IF EXISTS get_publisher_by_id; @@ -51,6 +54,7 @@ DROP TABLE IF EXISTS hierarchyid_composite_pk_table; DROP TABLE IF EXISTS hierarchyid_unique_table; DROP TABLE IF EXISTS unique_key_geometry_table; DROP TABLE IF EXISTS decimal_identity_geometry_table; +DROP TABLE IF EXISTS pk_and_unique_geometry_table; -- System versioning has to be released before the temporal table can be dropped. IF OBJECT_ID('dbo.temporal_geometry_type_table', 'U') IS NOT NULL AND OBJECTPROPERTY(OBJECT_ID('dbo.temporal_geometry_type_table'), 'TableTemporalType') = 2 @@ -318,6 +322,17 @@ CREATE TABLE unique_key_geometry_table( CONSTRAINT UQ_unique_key_geometry_table_code UNIQUE (code) ); +-- A primary key and a separate non-null unique key. A view over this table that does not select the +-- primary key is still keyable through the unique one, which is ordinary design rather than a key +-- the object cannot express. +CREATE TABLE pk_and_unique_geometry_table( + id int IDENTITY(1, 1) NOT NULL PRIMARY KEY, + code varchar(20) NOT NULL, + name varchar(100) NOT NULL, + geom geometry NULL, + CONSTRAINT UQ_pk_and_unique_geometry_table_code UNIQUE (code) +); + -- A decimal identity column. DataColumn.AutoIncrement would coerce its CLR type to Int32, which is -- why identity is carried from the catalog instead; this fixture is what proves the type survives. CREATE TABLE decimal_identity_geometry_table( @@ -756,6 +771,11 @@ VALUES ('CAR-001', 'point', geometry::STGeomFromText('POINT(1 2)', 0)), ('CAR-002', 'null geometry', NULL); +INSERT INTO pk_and_unique_geometry_table(code, name, geom) +VALUES + ('CAR-101', 'point', geometry::STGeomFromText('POINT(1 2)', 0)), + ('CAR-102', 'null geometry', NULL); + INSERT INTO decimal_identity_geometry_table(name, geom) VALUES ('point', geometry::STGeomFromText('POINT(1 2)', 0)), @@ -860,6 +880,14 @@ EXEC('CREATE VIEW books_publishers_view_composite_insertable as SELECT FROM dbo.books,dbo.publishers where publishers.id = books.publisher_id'); EXEC('CREATE VIEW geometry_only_view AS SELECT geom FROM dbo.geometry_type_table'); +-- A view has no index of its own, so its key can only be resolved through the underlying table. +EXEC('CREATE VIEW geometry_all_view AS SELECT id, name, geom FROM dbo.geometry_type_table'); +-- The underlying key is composite and one of its members is of an unsupported type, so no key the +-- projection can express identifies a row. Reporting the readable member alone would be a partial +-- key, which silently matches more than one row on an update or a delete. +EXEC('CREATE VIEW hierarchyid_composite_view AS SELECT tenant_id, name, node FROM dbo.hierarchyid_composite_pk_table'); +-- Omits the primary key of the underlying table and carries its unique key whole. +EXEC('CREATE VIEW unique_without_pk_view AS SELECT code, name, geom FROM dbo.pk_and_unique_geometry_table'); EXEC('CREATE PROCEDURE get_book_by_id @id int AS SELECT * FROM dbo.books WHERE id = @id'); diff --git a/src/Service.Tests/Snapshots/ConfigurationTests.TestReadingRuntimeConfigForMsSql.verified.txt b/src/Service.Tests/Snapshots/ConfigurationTests.TestReadingRuntimeConfigForMsSql.verified.txt index b0ce94e022..93c95abb72 100644 --- a/src/Service.Tests/Snapshots/ConfigurationTests.TestReadingRuntimeConfigForMsSql.verified.txt +++ b/src/Service.Tests/Snapshots/ConfigurationTests.TestReadingRuntimeConfigForMsSql.verified.txt @@ -1936,6 +1936,38 @@ ] } }, + { + TemporalGeometryType: { + Source: { + Object: temporal_geometry_type_table, + Type: Table + }, + GraphQL: { + Singular: TemporalGeometryType, + Plural: TemporalGeometryTypes, + Enabled: true + }, + Rest: { + Enabled: true + }, + Permissions: [ + { + Role: anonymous, + Actions: [ + { + Action: Create + }, + { + Action: Read + }, + { + Action: Update + } + ] + } + ] + } + }, { Profile: { Source: { diff --git a/src/Service.Tests/SqlTests/RestApiTests/Put/MsSqlPutApiTests.cs b/src/Service.Tests/SqlTests/RestApiTests/Put/MsSqlPutApiTests.cs index e9ea04a971..63d9bae9ec 100644 --- a/src/Service.Tests/SqlTests/RestApiTests/Put/MsSqlPutApiTests.cs +++ b/src/Service.Tests/SqlTests/RestApiTests/Put/MsSqlPutApiTests.cs @@ -125,6 +125,12 @@ public class MsSqlPutApiTests : PutApiTestBase $"AND [copies_sold] = 100 AND [last_sold_on] is NULL AND [last_sold_on_date] is NULL AND [row_version] is NOT NULL " + $"FOR JSON PATH, INCLUDE_NULL_VALUES, WITHOUT_ARRAY_WRAPPER" }, + { + "PutOneUpdateWithHiddenPeriodColumns", + $"SELECT [id], [name] FROM temporal_geometry_type_table " + + $"WHERE [id] = 1 AND [name] = 'Renamed collar' " + + $"FOR JSON PATH, INCLUDE_NULL_VALUES, WITHOUT_ARRAY_WRAPPER" + }, { "PutOneInsertWithRowversionFieldMissingFromRequestBody", $"SELECT * FROM {_tableWithReadOnlyFields } WHERE [id] = 2 AND [book_name] = 'Best seller' " + @@ -327,9 +333,37 @@ public async Task PutOneInViewBadRequest() await base.PutOneInViewBadRequest(expectedErrorMessage); } + /// + /// Test to validate successful execution of a PUT against a system-versioned temporal table + /// whose period columns are declared GENERATED ALWAYS ... HIDDEN. + /// PUT has overwrite semantics and nulls the writable columns a request body leaves out, so a + /// period column that reached the exposed contract would be targeted and the request would + /// fail with "Cannot update GENERATED ALWAYS columns". Successful execution confirms those + /// columns are not part of the contract: "SELECT *" does not return them, and schema + /// discovery does not name them either. + /// + [TestMethod] + public async Task PutOneWithHiddenPeriodColumnsInTemporalTable() + { + string requestBody = @" + { + ""name"": ""Renamed collar"" + }"; + + await SetupAndRunRestApiTest( + primaryKeyRoute: "id/1", + queryString: null, + entityNameOrPath: "TemporalGeometryType", + sqlQuery: GetQuery("PutOneUpdateWithHiddenPeriodColumns"), + operationType: EntityActionOperation.Upsert, + requestBody: requestBody, + expectedStatusCode: HttpStatusCode.OK + ); + } + /// /// Test to validate successful execution of a request when a rowversion field is missing from the request body. - /// In such a case, we don't attempt to NULL out rowversion field (as per PUT semantics) but instead skip updating/inserting the field. + /// In such a case, we don't attempt to NULL out rowversion field (as per PUT semantics) but instead skip updating/inserting the field. /// [TestMethod] public async Task PutOneWithRowversionFieldMissingFromRequestBody() diff --git a/src/Service.Tests/UnitTests/SqlMetadataProviderUnitTests.cs b/src/Service.Tests/UnitTests/SqlMetadataProviderUnitTests.cs index cc79832e0f..9dec2545ab 100644 --- a/src/Service.Tests/UnitTests/SqlMetadataProviderUnitTests.cs +++ b/src/Service.Tests/UnitTests/SqlMetadataProviderUnitTests.cs @@ -426,9 +426,9 @@ public async Task ValidateUnsupportedColumnTypeIsNotInferred() sourceDefinition.Columns.ContainsKey("geom"), message: "A column whose data type cannot be mapped is not expected in the source definition."); - // The narrowed path reads the shape without CommandBehavior.KeyInfo, so the reader - // reports no auto-increment flag and it is carried from the catalog instead. Asserted - // here because losing it would silently make creates require an identity value. + // Identity is carried from the catalog on the narrowed path rather than through + // DataColumn.AutoIncrement. Asserted here because losing it would silently make creates + // require an identity value. Assert.IsTrue( sourceDefinition.Columns["id"].IsAutoGenerated, message: "The identity column is expected to be marked auto-generated."); @@ -900,6 +900,336 @@ await SetUpSingleEntityMetadataProviderAsync( TestHelper.UnsetAllDABEnvironmentVariables(); } + /// + /// Test to validate that a database policy whose text merely contains "@item." inside a + /// string literal is not read as a field reference. + /// The policy below compares a literal to itself, so the two occurrences of "@item.geom" are + /// data, not column references, and the entity has to start. + /// + [TestMethod, TestCategory(TestCategory.MSSQL)] + public async Task ValidateQuotedLiteralInDatabasePolicyIsNotAFieldReference() + { + DatabaseEngine = TestCategory.MSSQL; + TestHelper.SetupDatabaseEnvironment(DatabaseEngine); + + Entity entity = new( + Source: new("dbo.geometry_type_table", EntitySourceType.Table, null, new string[] { "id" }), + Fields: null, + Rest: new(Enabled: true), + GraphQL: new("GeometryQuotedPolicy", "GeometryQuotedPolicys", Enabled: true), + Permissions: new EntityPermission[] + { + new(Role: "anonymous", + Actions: new EntityAction[] + { + new(Action: EntityActionOperation.Read, + Fields: null, + Policy: new(Request: null, Database: "@item.id gt 0 and '@item.geom' eq '@item.geom'")) + }) + }, + Relationships: null, + Mappings: null); + + await SetUpSingleEntityMetadataProviderAsync("GeometryQuotedPolicy", entity); + + await _sqlMetadataProvider.InitializeAsync(); + + Assert.IsTrue( + _sqlMetadataProvider.GetEntityNamesAndDbObjects().ContainsKey("GeometryQuotedPolicy"), + message: "A policy containing the field prefix only inside a string literal must not be read as a reference."); + + TestHelper.UnsetAllDABEnvironmentVariables(); + } + + /// + /// Test to validate that a policy referencing an alias of a supported column is honored even + /// when the alias carries the name of a skipped column. + /// Policy identifiers are exposed names, so "geom" here is the alias of the supported "name" + /// column and not the skipped spatial column that happens to share the name. + /// + [TestMethod, TestCategory(TestCategory.MSSQL)] + public async Task ValidateDatabasePolicyOverAliasOfSupportedColumnIsHonored() + { + DatabaseEngine = TestCategory.MSSQL; + TestHelper.SetupDatabaseEnvironment(DatabaseEngine); + + Entity entity = new( + Source: new("dbo.geometry_type_table", EntitySourceType.Table, null, new string[] { "id" }), + Fields: new List { new() { Name = "name", Alias = "geom" } }, + Rest: new(Enabled: true), + GraphQL: new("GeometryAliasPolicy", "GeometryAliasPolicys", Enabled: true), + Permissions: new EntityPermission[] + { + new(Role: "anonymous", + Actions: new EntityAction[] + { + new(Action: EntityActionOperation.Read, + Fields: null, + Policy: new(Request: null, Database: "@item.geom ne null")) + }) + }, + Relationships: null, + Mappings: null); + + await SetUpSingleEntityMetadataProviderAsync("GeometryAliasPolicy", entity); + + await _sqlMetadataProvider.InitializeAsync(); + + Assert.IsTrue( + _sqlMetadataProvider.GetEntityNamesAndDbObjects().ContainsKey("GeometryAliasPolicy"), + message: "A policy referencing an alias of a supported column must be honored, whatever the alias is named."); + + TestHelper.UnsetAllDABEnvironmentVariables(); + } + + /// + /// Test to validate that a simple view over a table holding an unsupported column keeps the + /// key inference it had before the projection was narrowed. + /// A view has no index of its own, so the catalog holds no key for it; the data adapter + /// resolved one through the underlying table, and describing the projection recovers the + /// same information without reintroducing the columns the projection left out. + /// + [TestMethod, TestCategory(TestCategory.MSSQL)] + public async Task ValidateViewKeyIsInferredWithoutConfiguredKeyFields() + { + DatabaseEngine = TestCategory.MSSQL; + TestHelper.SetupDatabaseEnvironment(DatabaseEngine); + + await SetUpSingleEntityMetadataProviderAsync( + "GeometryAllView", + BuildReadOnlyEntity( + entityName: "GeometryAllView", + databaseObject: "dbo.geometry_all_view", + sourceType: EntitySourceType.View, + keyFields: null)); + + await _sqlMetadataProvider.InitializeAsync(); + + Assert.IsTrue( + _sqlMetadataProvider.GetEntityNamesAndDbObjects().TryGetValue("GeometryAllView", out DatabaseObject databaseObject), + message: "Metadata inference failed for a simple view with no configured key."); + + SourceDefinition sourceDefinition = databaseObject.SourceDefinition; + + CollectionAssert.AreEqual( + new List { "id" }, + sourceDefinition.PrimaryKey, + message: "The key of the underlying table is expected to be inferred for the view."); + Assert.IsTrue( + sourceDefinition.Columns.ContainsKey("name"), + message: "A column with a supported data type is expected in the source definition."); + Assert.IsFalse( + sourceDefinition.Columns.ContainsKey("geom"), + message: "A column whose data type cannot be mapped is not expected in the source definition."); + + TestHelper.UnsetAllDABEnvironmentVariables(); + } + + /// + /// Test to validate that a view whose underlying key has a member the projection cannot + /// express does not load with a partial key. + /// Reporting only the readable member would produce a key that does not identify a row, and + /// a partial key silently matches more than one row on an update or a delete — worse than + /// the failure it would replace. + /// + [TestMethod, TestCategory(TestCategory.MSSQL)] + public async Task ValidateViewOverPartiallyUnsupportedKeyDoesNotInferPartialKey() + { + DatabaseEngine = TestCategory.MSSQL; + TestHelper.SetupDatabaseEnvironment(DatabaseEngine); + + await SetUpSingleEntityMetadataProviderAsync( + "HierarchyIdCompositeView", + BuildReadOnlyEntity( + entityName: "HierarchyIdCompositeView", + databaseObject: "dbo.hierarchyid_composite_view", + sourceType: EntitySourceType.View, + keyFields: null)); + + try + { + await _sqlMetadataProvider.InitializeAsync(); + Assert.Fail("Expected DataApiBuilderException was not thrown for a view whose underlying key cannot be expressed by the projection."); + } + catch (DataApiBuilderException ex) + { + Assert.AreEqual(HttpStatusCode.ServiceUnavailable, ex.StatusCode); + Assert.AreEqual(DataApiBuilderException.SubStatusCodes.ErrorInInitialization, ex.SubStatusCode); + Assert.IsTrue( + ex.Message.Contains("node"), + message: "The error is expected to name the key member the object does not expose, rather than " + + $"report a missing primary key. Actual message: {ex.Message}"); + } + + TestHelper.UnsetAllDABEnvironmentVariables(); + } + + /// + /// Test to validate that a view which does not select the underlying primary key is still + /// keyed through a unique key it does carry whole. + /// Unlike a table, where a key column absent from the projection means it was dropped as + /// unsupported, a view may simply not select the primary key. That is ordinary design, not a + /// key the object cannot express, so the search continues into the unique keys. + /// + [TestMethod, TestCategory(TestCategory.MSSQL)] + public async Task ValidateViewWithoutPrimaryKeyFallsBackToUniqueKey() + { + DatabaseEngine = TestCategory.MSSQL; + TestHelper.SetupDatabaseEnvironment(DatabaseEngine); + + await SetUpSingleEntityMetadataProviderAsync( + "UniqueWithoutPkView", + BuildReadOnlyEntity( + entityName: "UniqueWithoutPkView", + databaseObject: "dbo.unique_without_pk_view", + sourceType: EntitySourceType.View, + keyFields: null)); + + await _sqlMetadataProvider.InitializeAsync(); + + Assert.IsTrue( + _sqlMetadataProvider.GetEntityNamesAndDbObjects().TryGetValue("UniqueWithoutPkView", out DatabaseObject databaseObject), + message: "Metadata inference failed for a view that carries a unique key but not the primary key."); + + SourceDefinition sourceDefinition = databaseObject.SourceDefinition; + + CollectionAssert.AreEqual( + new List { "code" }, + sourceDefinition.PrimaryKey, + message: "The unique key the view carries whole is expected to be the key in effect."); + Assert.IsFalse( + sourceDefinition.Columns.ContainsKey("geom"), + message: "A column whose data type cannot be mapped is not expected in the source definition."); + + TestHelper.UnsetAllDABEnvironmentVariables(); + } + + /// + /// Test to validate that a doubled quote inside a policy literal is read as an escaped quote + /// and does not end the literal, so the field prefix that follows stays data. + /// + [TestMethod, TestCategory(TestCategory.MSSQL)] + public async Task ValidateEscapedQuoteInDatabasePolicyLiteralIsNotAFieldReference() + { + DatabaseEngine = TestCategory.MSSQL; + TestHelper.SetupDatabaseEnvironment(DatabaseEngine); + + Entity entity = new( + Source: new("dbo.geometry_type_table", EntitySourceType.Table, null, new string[] { "id" }), + Fields: null, + Rest: new(Enabled: true), + GraphQL: new("GeometryEscapedPolicy", "GeometryEscapedPolicys", Enabled: true), + Permissions: new EntityPermission[] + { + new(Role: "anonymous", + Actions: new EntityAction[] + { + new(Action: EntityActionOperation.Read, + Fields: null, + Policy: new(Request: null, Database: "@item.name ne 'it''s @item.geom'")) + }) + }, + Relationships: null, + Mappings: null); + + await SetUpSingleEntityMetadataProviderAsync("GeometryEscapedPolicy", entity); + + await _sqlMetadataProvider.InitializeAsync(); + + Assert.IsTrue( + _sqlMetadataProvider.GetEntityNamesAndDbObjects().ContainsKey("GeometryEscapedPolicy"), + message: "A doubled quote inside a literal must not end it, so the prefix that follows is not a reference."); + + TestHelper.UnsetAllDABEnvironmentVariables(); + } + + /// + /// Test to validate that the "fields" alias format is rejected when it names a column left + /// out of the projection, the same way "mappings" is. + /// + [TestMethod, TestCategory(TestCategory.MSSQL)] + public async Task ValidateFieldsAliasOverUnsupportedColumnFailsInitialization() + { + DatabaseEngine = TestCategory.MSSQL; + TestHelper.SetupDatabaseEnvironment(DatabaseEngine); + + Entity entity = new( + Source: new("dbo.geometry_type_table", EntitySourceType.Table, null, new string[] { "id" }), + Fields: new List { new() { Name = "geom", Alias = "Position" } }, + Rest: new(Enabled: true), + GraphQL: new("GeometryFieldsAlias", "GeometryFieldsAliass", Enabled: true), + Permissions: new EntityPermission[] + { + new(Role: "anonymous", + Actions: new EntityAction[] { new(Action: EntityActionOperation.Read, Fields: null, Policy: null) }) + }, + Relationships: null, + Mappings: null); + + await SetUpSingleEntityMetadataProviderAsync("GeometryFieldsAlias", entity); + + try + { + await _sqlMetadataProvider.InitializeAsync(); + Assert.Fail("Expected DataApiBuilderException was not thrown for a fields alias over a column of an unsupported data type."); + } + catch (DataApiBuilderException ex) + { + Assert.IsTrue( + ex.Message.Contains("geom") && ex.Message.Contains("fields"), + message: $"The error is expected to name the column and the configuration section. Actual message: {ex.Message}"); + } + + TestHelper.UnsetAllDABEnvironmentVariables(); + } + + /// + /// Test to validate that a permission's "fields.include" naming a column left out of the + /// projection fails initialization. + /// + [TestMethod, TestCategory(TestCategory.MSSQL)] + public async Task ValidateIncludedFieldOverUnsupportedColumnFailsInitialization() + { + DatabaseEngine = TestCategory.MSSQL; + TestHelper.SetupDatabaseEnvironment(DatabaseEngine); + + Entity entity = new( + Source: new("dbo.geometry_type_table", EntitySourceType.Table, null, new string[] { "id" }), + Fields: null, + Rest: new(Enabled: true), + GraphQL: new("GeometryIncluded", "GeometryIncludeds", Enabled: true), + Permissions: new EntityPermission[] + { + new(Role: "anonymous", + Actions: new EntityAction[] + { + new(Action: EntityActionOperation.Read, + Fields: new EntityActionFields( + Exclude: new HashSet(), + Include: new HashSet { "id", "name", "geom" }), + Policy: null) + }) + }, + Relationships: null, + Mappings: null); + + await SetUpSingleEntityMetadataProviderAsync("GeometryIncluded", entity); + + try + { + await _sqlMetadataProvider.InitializeAsync(); + Assert.Fail("Expected DataApiBuilderException was not thrown for a fields.include naming a column of an unsupported data type."); + } + catch (DataApiBuilderException ex) + { + Assert.IsTrue( + ex.Message.Contains("geom") && ex.Message.Contains("permissions"), + message: $"The error is expected to name the column and the configuration section. Actual message: {ex.Message}"); + } + + TestHelper.UnsetAllDABEnvironmentVariables(); + } + /// /// Builds a metadata provider over a single in-memory entity and resets the database state. /// The objects exercised by the unsupported-data-type tests are declared in memory rather diff --git a/src/Service.Tests/dab-config.MsSql.json b/src/Service.Tests/dab-config.MsSql.json index fc614d0e86..cd95b1fd28 100644 --- a/src/Service.Tests/dab-config.MsSql.json +++ b/src/Service.Tests/dab-config.MsSql.json @@ -2016,6 +2016,38 @@ } ] }, + "TemporalGeometryType": { + "source": { + "object": "temporal_geometry_type_table", + "type": "table" + }, + "graphql": { + "enabled": true, + "type": { + "singular": "TemporalGeometryType", + "plural": "TemporalGeometryTypes" + } + }, + "rest": { + "enabled": true + }, + "permissions": [ + { + "role": "anonymous", + "actions": [ + { + "action": "create" + }, + { + "action": "read" + }, + { + "action": "update" + } + ] + } + ] + }, "Profile": { "source": { "object": "profiles", From c07df8e5be67a5fa830dee13c7c9717b3e77c14b Mon Sep 17 00:00:00 2001 From: Joymax Andrade do Nascimento Date: Tue, 22 Sep 2026 00:15:24 -0300 Subject: [PATCH 14/19] Report an unreachable view key only when no key is in effect Two defects in the view key inference added in the previous commit, both found in review. The single-source check skipped hidden browse rows, and browse mode reports the key columns of every participating object. An object whose only selected column was dropped from the narrowed projection therefore survived in hidden rows alone, so a join read as a single object and was keyed by one side's primary key, which does not identify a row because the join can duplicate that side's rows. Every row with source information now counts toward the check; only selected columns carry the key. The unreachable-key failure was thrown where it was found, in the schema read that runs for every object, so a view whose source.key-fields is configured was rejected over a key it had been told not to use, with a message recommending exactly what was already configured. The reason is now returned, recorded per object, and reported from RejectUnreadablePrimaryKey, which only runs when no key is in effect. --- .../MsSqlMetadataProvider.cs | 42 ++++++---- .../MetadataProviders/SqlMetadataProvider.cs | 60 +++++++++++--- src/Service.Tests/DatabaseSchema-MsSql.sql | 7 ++ .../UnitTests/SqlMetadataProviderUnitTests.cs | 82 +++++++++++++++++++ 4 files changed, 164 insertions(+), 27 deletions(-) diff --git a/src/Core/Services/MetadataProviders/MsSqlMetadataProvider.cs b/src/Core/Services/MetadataProviders/MsSqlMetadataProvider.cs index 39885ee75a..a306eb8ca4 100644 --- a/src/Core/Services/MetadataProviders/MsSqlMetadataProvider.cs +++ b/src/Core/Services/MetadataProviders/MsSqlMetadataProvider.cs @@ -229,13 +229,14 @@ public MsSqlMetadataProvider( } /// - protected override async Task> GetProjectionKeyFromResultSetAsync(string selectStatement) + protected override async Task<(List Key, string? UnreachableKeyReason)> GetProjectionKeyFromResultSetAsync( + string selectStatement) { List projectionColumns = await DescribeProjectionAsync(selectStatement); if (projectionColumns.Count == 0) { - return new List(); + return (new List(), null); } // The key is taken from the underlying object's own catalog entry rather than from @@ -250,14 +251,18 @@ protected override async Task> GetProjectionKeyFromResultSetAsync(s foreach (ProjectionSourceColumn projectionColumn in projectionColumns) { - if (projectionColumn.IsHidden - || projectionColumn.SourceSchema is null + if (projectionColumn.SourceSchema is null || projectionColumn.SourceTable is null || projectionColumn.SourceColumn is null) { continue; } + // Every row counts toward this check, the hidden ones included. Browse mode reports + // the key columns of each participating object, so an object whose every selected + // column was dropped from the projection survives in hidden rows alone. Skipping + // those first would read a join as a single object and infer one side's key for a + // result the other side can duplicate rows of. if (sourceSchema is null) { sourceSchema = projectionColumn.SourceSchema; @@ -268,15 +273,20 @@ protected override async Task> GetProjectionKeyFromResultSetAsync(s { // More than one underlying object. Inferring a key across a join is out of // scope; such an entity needs source.key-fields, as it did before this change. - return new List(); + return (new List(), null); } - projectionBySourceColumn[projectionColumn.SourceColumn] = projectionColumn.Name; + // Only a column the projection selects can carry a key: a hidden one is exactly + // what the projection left out. + if (!projectionColumn.IsHidden) + { + projectionBySourceColumn[projectionColumn.SourceColumn] = projectionColumn.Name; + } } if (sourceSchema is null || sourceTable is null) { - return new List(); + return (new List(), null); } ObjectCatalogMetadata? sourceCatalogMetadata = @@ -284,7 +294,7 @@ protected override async Task> GetProjectionKeyFromResultSetAsync(s if (sourceCatalogMetadata is null) { - return new List(); + return (new List(), null); } // The underlying object's primary key comes first, then its unique keys, in index order @@ -321,7 +331,7 @@ protected override async Task> GetProjectionKeyFromResultSetAsync(s if (missingKeyColumn is null && projectionKey.Count > 0) { - return projectionKey; + return (projectionKey, null); } // Remembered from the first key that came closest, to name something concrete if no @@ -332,16 +342,18 @@ protected override async Task> GetProjectionKeyFromResultSetAsync(s if (unreachableKeyColumn is not null) { - throw new DataApiBuilderException( - message: $"No key of {sourceSchema}.{sourceTable}, which this object is built on, is fully exposed " + // Returned rather than thrown: an object whose source.key-fields is configured is + // reachable by that key, and rejecting it here would recommend exactly what was + // already configured. The caller reports this only if the object ends up keyless. + return ( + new List(), + $"No key of {sourceSchema}.{sourceTable}, which this object is built on, is fully exposed " + $"by it: the nearest one includes the column {unreachableKeyColumn}, which this object does " + "not expose. It therefore cannot be reached by key. Configure source.key-fields with columns " - + "it does expose that identify a row uniquely, if there are any.", - statusCode: HttpStatusCode.ServiceUnavailable, - subStatusCode: DataApiBuilderException.SubStatusCodes.ErrorInInitialization); + + "it does expose that identify a row uniquely, if there are any."); } - return new List(); + return (new List(), null); } /// diff --git a/src/Core/Services/MetadataProviders/SqlMetadataProvider.cs b/src/Core/Services/MetadataProviders/SqlMetadataProvider.cs index fd2b92cdd7..a9839f6e2e 100644 --- a/src/Core/Services/MetadataProviders/SqlMetadataProvider.cs +++ b/src/Core/Services/MetadataProviders/SqlMetadataProvider.cs @@ -124,6 +124,16 @@ public abstract class SqlMetadataProvider : /// private readonly ConcurrentDictionary _objectCatalogMetadataCache = new(StringComparer.Ordinal); + /// + /// Why no key of the object underneath a narrowed projection is reachable through it, for an + /// object the catalog holds no index for. Recorded rather than thrown where it is found, + /// because the schema read runs for every object while the answer only matters for one that + /// ends up with no key: an object whose source.key-fields is configured must not be rejected + /// over a key it was told not to use. Keyed by object with an ordinal comparer, for the + /// reason given on . + /// + private readonly ConcurrentDictionary _unreachableKeyReasonByObject = new(StringComparer.Ordinal); + /// /// The catalog facts an explicit schema projection needs and the "Columns" schema collection /// does not carry: which columns the database hides from "SELECT *", which columns identify @@ -191,15 +201,17 @@ protected sealed class ObjectCatalogMetadata /// A key column the projection left out must never be reported as part of a key: a partial /// key silently matches more than one row on an update or a delete. /// - /// - /// An implementation may fail here instead of returning empty, when it can tell that no key - /// of the underlying object is fully exposed and can name what is missing. That is a better - /// error than the generic missing-primary-key one, which reads as something the user forgot - /// to configure. - /// - protected virtual Task> GetProjectionKeyFromResultSetAsync(string selectStatement) + /// + /// The key, and a reason when the provider can tell that no key of the underlying object is + /// fully exposed and can name what is missing. The reason is only reported when the object + /// ends up with no key at all: a configured source.key-fields settles the question, and + /// failing here anyway would reject an object whose configuration already says how to reach + /// it — while recommending exactly what was configured. + /// + protected virtual Task<(List Key, string? UnreachableKeyReason)> GetProjectionKeyFromResultSetAsync( + string selectStatement) { - return Task.FromResult(new List()); + return Task.FromResult<(List, string?)>((new List(), null)); } /// @@ -1983,11 +1995,23 @@ private async Task FillSchemaForTableAsync( // asking for a CLR type. Only reached for an object the catalog could not key, and // resolved before the connection is opened for the same pooling reason as above. // Kept out of the cached metadata: it depends on this projection, not on the object. - List describedKey = catalogMetadata is not null + List describedKey = new(); + + if (catalogMetadata is not null && catalogMetadata.PrimaryKeyColumns.Count == 0 - && catalogMetadata.UniqueKeyCandidates.Count == 0 - ? await GetProjectionKeyFromResultSetAsync(selectStatement) - : new List(); + && catalogMetadata.UniqueKeyCandidates.Count == 0) + { + (List Key, string? UnreachableKeyReason) described = + await GetProjectionKeyFromResultSetAsync(selectStatement); + + describedKey = described.Key; + + if (described.UnreachableKeyReason is not null) + { + _unreachableKeyReasonByObject[GetObjectCacheKey(schemaName, tableName)] = + described.UnreachableKeyReason; + } + } await conn.OpenAsync(); @@ -2364,6 +2388,17 @@ private void RejectUnreadablePrimaryKey(string schemaName, string tableName) { string cacheKey = GetObjectCacheKey(schemaName, tableName); + // Reached only with no key in effect, which is what makes this the right place to report + // it: the schema read found that no key of the object underneath is reachable through + // this projection, and nothing configured has since supplied one. + if (_unreachableKeyReasonByObject.TryGetValue(cacheKey, out string? unreachableKeyReason)) + { + throw new DataApiBuilderException( + message: unreachableKeyReason, + statusCode: HttpStatusCode.ServiceUnavailable, + subStatusCode: DataApiBuilderException.SubStatusCodes.ErrorInInitialization); + } + if (!_skippedColumnsByObject.TryGetValue(cacheKey, out Dictionary? skippedColumns) || !_objectCatalogMetadataCache.TryGetValue(cacheKey, out ObjectCatalogMetadata? catalogMetadata)) { @@ -2634,6 +2669,7 @@ private void ReleaseCatalogMetadataCaches() _columnsMetadataCache.Clear(); _skippedColumnsByObject.Clear(); _objectCatalogMetadataCache.Clear(); + _unreachableKeyReasonByObject.Clear(); } /// diff --git a/src/Service.Tests/DatabaseSchema-MsSql.sql b/src/Service.Tests/DatabaseSchema-MsSql.sql index 65a668dbbb..d993d1863e 100644 --- a/src/Service.Tests/DatabaseSchema-MsSql.sql +++ b/src/Service.Tests/DatabaseSchema-MsSql.sql @@ -15,6 +15,7 @@ DROP VIEW IF EXISTS geometry_only_view; DROP VIEW IF EXISTS geometry_all_view; DROP VIEW IF EXISTS hierarchyid_composite_view; DROP VIEW IF EXISTS unique_without_pk_view; +DROP VIEW IF EXISTS join_geometry_view; DROP PROCEDURE IF EXISTS get_books; DROP PROCEDURE IF EXISTS get_book_by_id; DROP PROCEDURE IF EXISTS get_publisher_by_id; @@ -888,6 +889,12 @@ EXEC('CREATE VIEW geometry_all_view AS SELECT id, name, geom FROM dbo.geometry_t EXEC('CREATE VIEW hierarchyid_composite_view AS SELECT tenant_id, name, node FROM dbo.hierarchyid_composite_pk_table'); -- Omits the primary key of the underlying table and carries its unique key whole. EXEC('CREATE VIEW unique_without_pk_view AS SELECT code, name, geom FROM dbo.pk_and_unique_geometry_table'); +-- A join whose second object contributes only the unsupported column. Once that column is dropped +-- from the projection, every column the projection selects resolves to the first object, and only +-- the hidden key column browse mode reports for the second one reveals that this is a join at all. +-- A key inferred from the first object alone would not identify a row, since the join can duplicate +-- its rows. +EXEC('CREATE VIEW join_geometry_view AS SELECT a.id, a.name, b.geom FROM dbo.geometry_type_table a JOIN dbo.unique_key_geometry_table b ON b.name = a.name'); EXEC('CREATE PROCEDURE get_book_by_id @id int AS SELECT * FROM dbo.books WHERE id = @id'); diff --git a/src/Service.Tests/UnitTests/SqlMetadataProviderUnitTests.cs b/src/Service.Tests/UnitTests/SqlMetadataProviderUnitTests.cs index 9dec2545ab..4587e4589e 100644 --- a/src/Service.Tests/UnitTests/SqlMetadataProviderUnitTests.cs +++ b/src/Service.Tests/UnitTests/SqlMetadataProviderUnitTests.cs @@ -1104,6 +1104,88 @@ await SetUpSingleEntityMetadataProviderAsync( TestHelper.UnsetAllDABEnvironmentVariables(); } + /// + /// Test to validate that a view built on a join does not have one side's key inferred as its + /// own, even when every column the projection selects resolves to that side. + /// The second object of this view contributes only the unsupported column, so once that + /// column is dropped nothing the projection selects points at it. Browse mode still reports + /// its key column, flagged hidden, which is what identifies the result as a join. A key taken + /// from the first object alone would not identify a row, because the join can duplicate its + /// rows — the same class of defect as a partial key. + /// + [TestMethod, TestCategory(TestCategory.MSSQL)] + public async Task ValidateViewOverJoinDoesNotInferKeyFromOneSide() + { + DatabaseEngine = TestCategory.MSSQL; + TestHelper.SetupDatabaseEnvironment(DatabaseEngine); + + await SetUpSingleEntityMetadataProviderAsync( + "JoinGeometryView", + BuildReadOnlyEntity( + entityName: "JoinGeometryView", + databaseObject: "dbo.join_geometry_view", + sourceType: EntitySourceType.View, + keyFields: null)); + + try + { + await _sqlMetadataProvider.InitializeAsync(); + Assert.Fail("Expected DataApiBuilderException was not thrown for a view built on a join with no configured key."); + } + catch (DataApiBuilderException ex) + { + Assert.AreEqual(HttpStatusCode.ServiceUnavailable, ex.StatusCode); + Assert.AreEqual(DataApiBuilderException.SubStatusCodes.ErrorInInitialization, ex.SubStatusCode); + Assert.IsTrue( + ex.Message.Contains("Primary key not configured"), + message: "A join is out of scope for key inference, so the object is expected to require " + + $"source.key-fields exactly as it did before. Actual message: {ex.Message}"); + } + + TestHelper.UnsetAllDABEnvironmentVariables(); + } + + /// + /// Test to validate that a configured key loads a view whose underlying key the projection + /// cannot express. + /// The schema read runs for every object, so it also finds that no key of the object + /// underneath is reachable here. That finding must not reject an object whose + /// source.key-fields already says how to reach it — failing then would recommend precisely + /// what was configured. + /// + [TestMethod, TestCategory(TestCategory.MSSQL)] + public async Task ValidateConfiguredKeyIsHonoredWhenViewOmitsUnderlyingKey() + { + DatabaseEngine = TestCategory.MSSQL; + TestHelper.SetupDatabaseEnvironment(DatabaseEngine); + + await SetUpSingleEntityMetadataProviderAsync( + "HierarchyIdCompositeViewWithKey", + BuildReadOnlyEntity( + entityName: "HierarchyIdCompositeViewWithKey", + databaseObject: "dbo.hierarchyid_composite_view", + sourceType: EntitySourceType.View, + keyFields: new[] { "tenant_id" })); + + await _sqlMetadataProvider.InitializeAsync(); + + Assert.IsTrue( + _sqlMetadataProvider.GetEntityNamesAndDbObjects().TryGetValue("HierarchyIdCompositeViewWithKey", out DatabaseObject databaseObject), + message: "Metadata inference failed for a view whose key is configured through source.key-fields."); + + SourceDefinition sourceDefinition = databaseObject.SourceDefinition; + + CollectionAssert.AreEqual( + new List { "tenant_id" }, + sourceDefinition.PrimaryKey, + message: "The configured key is expected to be the key in effect."); + Assert.IsFalse( + sourceDefinition.Columns.ContainsKey("node"), + message: "A column whose data type cannot be mapped is not expected in the source definition."); + + TestHelper.UnsetAllDABEnvironmentVariables(); + } + /// /// Test to validate that a doubled quote inside a policy literal is read as an escaped quote /// and does not end the literal, so the field prefix that follows stays data. From 79865db158e4aa8c0c5ca6da2ac7ff5b41b281f3 Mon Sep 17 00:00:00 2001 From: Joymax Andrade do Nascimento Date: Tue, 22 Sep 2026 01:17:56 -0300 Subject: [PATCH 15/19] Decline view key inference for a self-join and check relationship fields Two more findings from review. Browse mode reports the physical table rather than the alias, so the single-source check cannot see a self-join: both instances of the object look like one, and the object's key was inferred for a result whose rows the join can duplicate. A source column reported twice is what gives it away, since each instance contributes the key, so inference is declined when one repeats. The rule is conservative: a projection selecting one column twice under different names is declined too, and needs source.key-fields as it did before. Configured relationships were the one place a reference to a skipped column was still accepted. The exposed and backing column maps are built from the configuration rather than from the source definition, so the reference kept resolving and failed per request instead, in MultipleCreateOrderHelper, which indexes SourceDefinition.Columns by each relationship field. The check runs after every object definition is populated, because a relationship's target.fields belong to the other entity's object. Foreign keys inferred from the catalog are left alone: there is no configuration to correct there, so refusing to start would punish the database's shape rather than the configuration. --- .../MsSqlMetadataProvider.cs | 14 ++ .../MetadataProviders/SqlMetadataProvider.cs | 85 ++++++++++++ src/Service.Tests/DatabaseSchema-MsSql.sql | 5 + .../UnitTests/SqlMetadataProviderUnitTests.cs | 122 +++++++++++++++++- 4 files changed, 224 insertions(+), 2 deletions(-) diff --git a/src/Core/Services/MetadataProviders/MsSqlMetadataProvider.cs b/src/Core/Services/MetadataProviders/MsSqlMetadataProvider.cs index a306eb8ca4..5026cdccbf 100644 --- a/src/Core/Services/MetadataProviders/MsSqlMetadataProvider.cs +++ b/src/Core/Services/MetadataProviders/MsSqlMetadataProvider.cs @@ -248,6 +248,7 @@ public MsSqlMetadataProvider( string? sourceSchema = null; string? sourceTable = null; Dictionary projectionBySourceColumn = new(StringComparer.OrdinalIgnoreCase); + HashSet sourceColumnsSeen = new(StringComparer.OrdinalIgnoreCase); foreach (ProjectionSourceColumn projectionColumn in projectionColumns) { @@ -276,6 +277,19 @@ public MsSqlMetadataProvider( return (new List(), null); } + // A source column reported twice means the result draws on more than one instance + // of the object: a self-join, which the check above cannot see, because browse mode + // reports the physical table rather than the alias. Such a join duplicates rows of + // the object just as any other does, so its key does not identify a row of the + // result. Declining is conservative — a projection that selects one column twice + // under different names is refused too, and simply needs source.key-fields, as it + // did before this change — and the alternative is a key that silently matches more + // than one row on an update or a delete. + if (!sourceColumnsSeen.Add(projectionColumn.SourceColumn)) + { + return (new List(), null); + } + // Only a column the projection selects can carry a key: a hidden one is exactly // what the projection left out. if (!projectionColumn.IsHidden) diff --git a/src/Core/Services/MetadataProviders/SqlMetadataProvider.cs b/src/Core/Services/MetadataProviders/SqlMetadataProvider.cs index a9839f6e2e..896d1c854e 100644 --- a/src/Core/Services/MetadataProviders/SqlMetadataProvider.cs +++ b/src/Core/Services/MetadataProviders/SqlMetadataProvider.cs @@ -1373,6 +1373,17 @@ private async Task PopulateObjectDefinitionForEntities() await PopulateObjectDefinitionForEntity(entityName, entity); } + try + { + // After the loops above, not inside them: a relationship's "target.fields" belong + // to the other entity's object, which may not have been read yet while this one is. + RejectConfiguredRelationshipReferencesToSkippedColumns(); + } + catch (Exception e) + { + HandleOrRecordException(e); + } + try { await PopulateForeignKeyDefinitionAsync(); @@ -2599,6 +2610,80 @@ private static IEnumerable EnumeratePolicyFieldReferences(string? databa } } + /// + /// Fails initialization when a configured relationship names a column that was left out of + /// a projection because its data type is not supported. Runs after every object definition + /// is populated, unlike the other reference checks: a relationship's "target.fields" belong + /// to the other entity's object, which may not have been read yet while this one is. + /// The reference would otherwise keep resolving — the exposed and backing column maps are + /// built from the configuration rather than from + /// — and fail per request instead, in + /// MultipleCreateOrderHelper, which indexes that dictionary by each relationship field. + /// Foreign keys inferred from the catalog are deliberately not checked: there is no + /// configuration to correct there, so refusing to start would punish the database's shape + /// rather than the configuration, and the relationship simply not forming is the + /// proportionate outcome. + /// + private void RejectConfiguredRelationshipReferencesToSkippedColumns() + { + foreach ((string entityName, Entity entity) in Entities) + { + if (entity.Relationships is null) + { + continue; + } + + foreach ((string relationshipName, EntityRelationship relationship) in entity.Relationships) + { + RejectRelationshipFields( + entityName, + relationship.SourceFields, + $"source.fields of relationship {relationshipName} of entity {entityName}"); + + RejectRelationshipFields( + relationship.TargetEntity, + relationship.TargetFields, + $"target.fields of relationship {relationshipName} of entity {entityName}"); + } + } + } + + /// + /// Fails initialization when any of the given fields, read as names configured on the given + /// entity, resolves to a column that entity's object does not expose. + /// + private void RejectRelationshipFields(string entityName, string[]? fields, string configurationSection) + { + if (fields is null + || fields.Length == 0 + || !Entities.TryGetValue(entityName, out Entity? entity) + || !EntityToDatabaseObject.TryGetValue(entityName, out DatabaseObject? databaseObject) + || !_skippedColumnsByObject.TryGetValue( + GetObjectCacheKey(databaseObject.SchemaName, databaseObject.Name), + out Dictionary? skippedColumns)) + { + return; + } + + foreach (string field in fields) + { + string backingColumn = ResolveBackingColumnName(entity, field); + + if (!skippedColumns.TryGetValue(backingColumn, out string? dataType)) + { + continue; + } + + throw new DataApiBuilderException( + message: $"The {configurationSection} reference the column {backingColumn} of " + + $"{databaseObject.SchemaName}.{databaseObject.Name}, whose data type {dataType} is not " + + "supported. That column is not part of the exposed contract, so the relationship cannot be " + + "honored. Remove it from the configuration.", + statusCode: HttpStatusCode.ServiceUnavailable, + subStatusCode: DataApiBuilderException.SubStatusCodes.ErrorInInitialization); + } + } + /// /// Resolves an exposed field name to the column it is backed by, using the aliases the /// configuration declares through "mappings" and through a field's "alias". An unaliased diff --git a/src/Service.Tests/DatabaseSchema-MsSql.sql b/src/Service.Tests/DatabaseSchema-MsSql.sql index d993d1863e..f91c1ba0cd 100644 --- a/src/Service.Tests/DatabaseSchema-MsSql.sql +++ b/src/Service.Tests/DatabaseSchema-MsSql.sql @@ -16,6 +16,7 @@ DROP VIEW IF EXISTS geometry_all_view; DROP VIEW IF EXISTS hierarchyid_composite_view; DROP VIEW IF EXISTS unique_without_pk_view; DROP VIEW IF EXISTS join_geometry_view; +DROP VIEW IF EXISTS self_join_geometry_view; DROP PROCEDURE IF EXISTS get_books; DROP PROCEDURE IF EXISTS get_book_by_id; DROP PROCEDURE IF EXISTS get_publisher_by_id; @@ -895,6 +896,10 @@ EXEC('CREATE VIEW unique_without_pk_view AS SELECT code, name, geom FROM dbo.pk_ -- A key inferred from the first object alone would not identify a row, since the join can duplicate -- its rows. EXEC('CREATE VIEW join_geometry_view AS SELECT a.id, a.name, b.geom FROM dbo.geometry_type_table a JOIN dbo.unique_key_geometry_table b ON b.name = a.name'); +-- A self-join, where browse mode reports the same physical table for both instances and so cannot +-- be told apart from a single one by source lineage alone. What does give it away is the primary +-- key column being reported twice, once for each instance. +EXEC('CREATE VIEW self_join_geometry_view AS SELECT a.id, a.name, b.geom FROM dbo.geometry_type_table a JOIN dbo.geometry_type_table b ON b.name = a.name'); EXEC('CREATE PROCEDURE get_book_by_id @id int AS SELECT * FROM dbo.books WHERE id = @id'); diff --git a/src/Service.Tests/UnitTests/SqlMetadataProviderUnitTests.cs b/src/Service.Tests/UnitTests/SqlMetadataProviderUnitTests.cs index 4587e4589e..a674dad87a 100644 --- a/src/Service.Tests/UnitTests/SqlMetadataProviderUnitTests.cs +++ b/src/Service.Tests/UnitTests/SqlMetadataProviderUnitTests.cs @@ -1145,6 +1145,115 @@ await SetUpSingleEntityMetadataProviderAsync( TestHelper.UnsetAllDABEnvironmentVariables(); } + /// + /// Test to validate that a relationship naming a column left out of the projection fails + /// initialization. + /// The exposed and backing column maps are built from the configuration rather than from + /// the source definition, so the reference keeps resolving after the column is gone and + /// fails per request instead — in MultipleCreateOrderHelper, which indexes + /// by each relationship field. + /// + [TestMethod, TestCategory(TestCategory.MSSQL)] + public async Task ValidateRelationshipFieldOverUnsupportedColumnFailsInitialization() + { + DatabaseEngine = TestCategory.MSSQL; + TestHelper.SetupDatabaseEnvironment(DatabaseEngine); + + Entity sourceEntity = BuildReadOnlyEntity( + entityName: "UniqueKeyGeometrySource", + databaseObject: "dbo.unique_key_geometry_table", + sourceType: EntitySourceType.Table, + keyFields: null) + with + { + Relationships = new Dictionary + { + { + "geometryType", + new EntityRelationship( + Cardinality: Cardinality.One, + TargetEntity: "GeometryTypeTarget", + SourceFields: new[] { "geom" }, + TargetFields: new[] { "id" }, + LinkingObject: null, + LinkingSourceFields: Array.Empty(), + LinkingTargetFields: Array.Empty()) + } + } + }; + + await SetUpEntitiesMetadataProviderAsync(new Dictionary + { + { "UniqueKeyGeometrySource", sourceEntity }, + { + "GeometryTypeTarget", + BuildReadOnlyEntity( + entityName: "GeometryTypeTarget", + databaseObject: "dbo.geometry_type_table", + sourceType: EntitySourceType.Table, + keyFields: null) + } + }); + + try + { + await _sqlMetadataProvider.InitializeAsync(); + Assert.Fail("Expected DataApiBuilderException was not thrown for a relationship naming a column whose data type is not supported."); + } + catch (DataApiBuilderException ex) + { + Assert.AreEqual(HttpStatusCode.ServiceUnavailable, ex.StatusCode); + Assert.AreEqual(DataApiBuilderException.SubStatusCodes.ErrorInInitialization, ex.SubStatusCode); + Assert.IsTrue( + ex.Message.Contains("geom"), + message: "The error is expected to name the column the relationship cannot be honored over. " + + $"Actual message: {ex.Message}"); + } + + TestHelper.UnsetAllDABEnvironmentVariables(); + } + + /// + /// Test to validate that a view built on a self-join does not have the object's key + /// inferred as its own. + /// Browse mode reports the physical table rather than the alias, so both instances of the + /// object look like one and the single-source check cannot see the join. What gives it away + /// is the key column being reported twice, once per instance. A self-join duplicates rows + /// of the object just as any other join does, so its key does not identify a row of the + /// result. + /// + [TestMethod, TestCategory(TestCategory.MSSQL)] + public async Task ValidateViewOverSelfJoinDoesNotInferKey() + { + DatabaseEngine = TestCategory.MSSQL; + TestHelper.SetupDatabaseEnvironment(DatabaseEngine); + + await SetUpSingleEntityMetadataProviderAsync( + "SelfJoinGeometryView", + BuildReadOnlyEntity( + entityName: "SelfJoinGeometryView", + databaseObject: "dbo.self_join_geometry_view", + sourceType: EntitySourceType.View, + keyFields: null)); + + try + { + await _sqlMetadataProvider.InitializeAsync(); + Assert.Fail("Expected DataApiBuilderException was not thrown for a view built on a self-join with no configured key."); + } + catch (DataApiBuilderException ex) + { + Assert.AreEqual(HttpStatusCode.ServiceUnavailable, ex.StatusCode); + Assert.AreEqual(DataApiBuilderException.SubStatusCodes.ErrorInInitialization, ex.SubStatusCode); + Assert.IsTrue( + ex.Message.Contains("Primary key not configured"), + message: "A self-join is out of scope for key inference, so the object is expected to require " + + $"source.key-fields exactly as it did before. Actual message: {ex.Message}"); + } + + TestHelper.UnsetAllDABEnvironmentVariables(); + } + /// /// Test to validate that a configured key loads a view whose underlying key the projection /// cannot express. @@ -1318,11 +1427,20 @@ public async Task ValidateIncludedFieldOverUnsupportedColumnFailsInitialization( /// than in dab-config.MsSql.json, because several of them fail by design and every MSSQL /// fixture initializes every configured entity. /// - private static async Task SetUpSingleEntityMetadataProviderAsync(string entityName, Entity entity) + private static Task SetUpSingleEntityMetadataProviderAsync(string entityName, Entity entity) + { + return SetUpEntitiesMetadataProviderAsync(new Dictionary { { entityName, entity } }); + } + + /// + /// Sets up a metadata provider over the given entities alone, declared in memory. Needed + /// wherever more than one entity is involved, such as a configured relationship. + /// + private static async Task SetUpEntitiesMetadataProviderAsync(Dictionary entities) { RuntimeConfig runtimeConfig = SqlTestHelper.SetupRuntimeConfig() with - { Entities = new RuntimeEntities(new Dictionary { { entityName, entity } }) }; + { Entities = new RuntimeEntities(entities) }; RuntimeConfigProvider runtimeConfigProvider = TestHelper.GenerateInMemoryRuntimeConfigProvider(runtimeConfig); SetUpSQLMetadataProvider(runtimeConfigProvider); await ResetDbStateAsync(); From fd332857db79c1cb9c16f5afb95def3e6dee0366 Mon Sep 17 00:00:00 2001 From: Joymax Andrade do Nascimento Date: Tue, 22 Sep 2026 01:44:48 -0300 Subject: [PATCH 16/19] Exclude hypothetical indexes from unique key candidates A hypothetical index exists only to hold column-level statistics and cannot be used as a data access path, so it enforces no uniqueness. The catalog query accepted one as a unique key candidate, which on a table with no primary key would key the object on columns nothing keeps unique, letting an update or a delete by key match more than one row. Excluded alongside disabled and filtered indexes, for the same reason. No fixture: the only way to create a hypothetical index is WITH STATISTICS_ONLY, undocumented syntax the Database Engine Tuning Advisor uses, and tying the test schema to it would break CI if it ever changes. Verified against SQL Server 2025 that such an index is reported with is_unique = 1 and is_hypothetical = 1, which is what made it eligible. --- .../Services/MetadataProviders/MsSqlMetadataProvider.cs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/Core/Services/MetadataProviders/MsSqlMetadataProvider.cs b/src/Core/Services/MetadataProviders/MsSqlMetadataProvider.cs index 5026cdccbf..3283e43345 100644 --- a/src/Core/Services/MetadataProviders/MsSqlMetadataProvider.cs +++ b/src/Core/Services/MetadataProviders/MsSqlMetadataProvider.cs @@ -84,7 +84,10 @@ public MsSqlMetadataProvider( // the data adapter reports a non-nullable unique key as DataTable.PrimaryKey on the // unnarrowed path, and the narrowed path has to do the same. Filtered and disabled // indexes do not identify every row, and an index's included columns report key_ordinal - // 0 and are not part of its key. + // 0 and are not part of its key. A hypothetical index is excluded for the same reason: + // it exists only to hold column-level statistics, cannot be used as a data access path, + // and therefore enforces no uniqueness at all, so keying an object on one would let an + // update or a delete by key match more than one row. string query = "select c.name as COLUMN_NAME, c.is_hidden as IS_HIDDEN, c.is_identity as IS_IDENTITY, " + "c.is_nullable as IS_NULLABLE, i.index_id as INDEX_ID, " @@ -93,7 +96,8 @@ public MsSqlMetadataProvider( + "left join sys.index_columns as ic on ic.object_id = c.object_id " + "and ic.column_id = c.column_id and ic.key_ordinal > 0 " + "left join sys.indexes as i on i.object_id = ic.object_id and i.index_id = ic.index_id " - + "and i.is_unique = 1 and i.is_disabled = 0 and i.has_filter = 0 " + + "and i.is_unique = 1 and i.is_disabled = 0 and i.is_hypothetical = 0 " + + "and i.has_filter = 0 " + $"where c.object_id = object_id(quotename({schemaParamName})+'.'+quotename({tableParamName}));"; Dictionary parameters = new() From 997d3f6e224b95780bd416eeb7141904e5c90557 Mon Sep 17 00:00:00 2001 From: Aaron Burtle Date: Wed, 11 Nov 2026 21:14:56 -0800 Subject: [PATCH 17/19] Skip columns the data provider can't read during schema discovery Keep FillSchema for every object it can describe. When it rejects an object because the provider reports no CLR type for a column (geometry, geography, hierarchyid and other CLR types), re-read the schema with SchemaOnly | KeyInfo and build the DataTable with the data adapter's column and key rules, without those columns. - Detect unreadable columns by their missing DataType rather than by type name, so Type System Version=SQL Server 2005 still exposes them as byte[] and custom CLR types are covered. - Fail with a specific error when the inferred key, or every column, is unreadable. - Validate configuration references to skipped columns once the exposed-to-backing maps exist: key-fields, mappings, fields, fields.include, database policies and relationship fields. - Track skipped columns per schema DataTable, which entities over the same object share whatever the casing of its name. - Remove the catalog-based projection and its queries. --- .../Authorization/AuthorizationResolver.cs | 56 + .../MsSqlMetadataProvider.cs | 430 ------- .../MetadataProviders/SqlMetadataProvider.cs | 1121 ++++------------- 3 files changed, 318 insertions(+), 1289 deletions(-) diff --git a/src/Core/Authorization/AuthorizationResolver.cs b/src/Core/Authorization/AuthorizationResolver.cs index 4c9fa1d3c6..6cae9cae1b 100644 --- a/src/Core/Authorization/AuthorizationResolver.cs +++ b/src/Core/Authorization/AuthorizationResolver.cs @@ -810,6 +810,62 @@ private static ResolvedDatabasePolicy GetPolicyWithClaimValues(string policy, Di return new ResolvedDatabasePolicy(processedPolicy, claimValues); } + /// + /// Returns the field names a database policy references through "@item.", skipping single-quoted + /// literals, in which a doubled quote is an escaped quote. + /// + internal static IEnumerable GetFieldReferencesInDatabasePolicy(string? policy) + { + if (string.IsNullOrEmpty(policy)) + { + yield break; + } + + int index = 0; + while (index < policy.Length) + { + if (policy[index] == '\'') + { + index++; + while (index < policy.Length) + { + if (policy[index++] != '\'') + { + continue; + } + + if (index < policy.Length && policy[index] == '\'') + { + index++; + continue; + } + + break; + } + + continue; + } + + if (string.CompareOrdinal(policy, index, FIELD_PREFIX, 0, FIELD_PREFIX.Length) != 0) + { + index++; + continue; + } + + int fieldStart = index + FIELD_PREFIX.Length; + index = fieldStart; + while (index < policy.Length && (char.IsAsciiLetterOrDigit(policy[index]) || policy[index] == '_')) + { + index++; + } + + if (index > fieldStart) + { + yield return policy[fieldStart..index]; + } + } + } + /// /// Emits a structured audit signal when a configured (non-empty) database authorization /// policy resolves to an empty predicate, which silently removes the intended row/column diff --git a/src/Core/Services/MetadataProviders/MsSqlMetadataProvider.cs b/src/Core/Services/MetadataProviders/MsSqlMetadataProvider.cs index ae2621a226..2e6e5b1ae2 100644 --- a/src/Core/Services/MetadataProviders/MsSqlMetadataProvider.cs +++ b/src/Core/Services/MetadataProviders/MsSqlMetadataProvider.cs @@ -1,7 +1,6 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -using System.Collections.Immutable; using System.Data; using System.Data.Common; using System.Net; @@ -49,435 +48,6 @@ public MsSqlMetadataProvider( _childConfig = childConfig; } - /// - /// SQL Server CLR user-defined types. Microsoft.Data.SqlClient resolves their CLR type - /// through the Microsoft.SqlServer.Types assembly, which Data API builder does not - /// reference, so the reader reports no type for the column and the data adapter fails. - /// Deliberately limited to the types that cannot be read at all: timestamp, xml and vector - /// columns do resolve to a CLR type and are left untouched. - /// - private static readonly ImmutableHashSet _unsupportedColumnDataTypes = - ImmutableHashSet.Create( - StringComparer.OrdinalIgnoreCase, - "geometry", - "geography", - "hierarchyid"); - - /// - protected override ImmutableHashSet UnsupportedColumnDataTypes => _unsupportedColumnDataTypes; - - /// - protected override async Task GetObjectCatalogMetadataAsync( - string schemaName, - string tableName) - { - string schemaParamName = $"{BaseQueryStructure.PARAM_NAME_PREFIX}param0"; - string tableParamName = $"{BaseQueryStructure.PARAM_NAME_PREFIX}param1"; - - // The object is resolved through object_id(). Both name parts go through QUOTENAME: - // object_id() parses its argument as a multi-part name, so an unquoted schema or table - // holding a dot, a space, a reserved word or a closing bracket resolves to the wrong - // object or to null — and a null object_id returns no rows, which would leave the - // projection at "SELECT *" and bring #3801 back for that object. QUOTENAME also doubles - // an embedded "]", so the names are passed raw and quoted by the server. - // is_hidden marks the period columns of a temporal table declared - // GENERATED ALWAYS ... HIDDEN, which "SELECT *" does not return. key_ordinal is null for - // every column outside the primary key, and 1-based within it; an index's included - // columns report 0 and are not part of the key. - // Unique indexes are returned alongside the primary key, because absent a primary key - // the data adapter reports a non-nullable unique key as DataTable.PrimaryKey on the - // unnarrowed path, and the narrowed path has to do the same. Filtered and disabled - // indexes do not identify every row, and an index's included columns report key_ordinal - // 0 and are not part of its key. A hypothetical index is excluded for the same reason: - // it exists only to hold column-level statistics, cannot be used as a data access path, - // and therefore enforces no uniqueness at all, so keying an object on one would let an - // update or a delete by key match more than one row. - string query = - "select c.name as COLUMN_NAME, c.is_hidden as IS_HIDDEN, c.is_identity as IS_IDENTITY, " - + "c.is_nullable as IS_NULLABLE, i.index_id as INDEX_ID, " - + "i.is_primary_key as IS_PRIMARY_KEY, ic.key_ordinal as KEY_ORDINAL " - + "from sys.columns as c " - + "left join sys.index_columns as ic on ic.object_id = c.object_id " - + "and ic.column_id = c.column_id and ic.key_ordinal > 0 " - + "left join sys.indexes as i on i.object_id = ic.object_id and i.index_id = ic.index_id " - + "and i.is_unique = 1 and i.is_disabled = 0 and i.is_hypothetical = 0 " - + "and i.has_filter = 0 " - + $"where c.object_id = object_id(quotename({schemaParamName})+'.'+quotename({tableParamName}));"; - - Dictionary parameters = new() - { - { schemaParamName, new(schemaName, DbType.String) }, - { tableParamName, new(tableName, DbType.String) } - }; - - try - { - return await QueryExecutor.ExecuteQueryAsync( - sqltext: query, - parameters: parameters, - dataReaderHandler: SummarizeObjectCatalogMetadataAsync, - dataSourceName: _dataSourceName); - } - catch (Exception ex) - { - // sys.columns.is_hidden exists from SQL Server 2016 on. Where the catalog cannot - // answer — a dedicated SQL pool, or a login without VIEW DEFINITION — returning null - // leaves the projection at "*", which is exactly the behavior before this change. - // Logged at Warning, not Debug: the fallback changes what the object exposes — the - // projection stays "SELECT *" and an unsupported column takes the entity down with - // the provider's own opaque error — so the reason has to be visible by default. - _logger.LogWarning( - "Unable to read catalog metadata for {schemaName}.{tableName}: {message}", - schemaName, - tableName, - ex.Message); - - return null; - } - } - - /// - /// Turns the catalog rows read by into - /// . - /// Returns null when the object has no rows: that means it was not found in the catalog, and - /// claiming it has no hidden columns would be a guess. - /// - private async Task SummarizeObjectCatalogMetadataAsync( - DbDataReader reader, - List? args = null) - { - DbResultSet catalogRows = await QueryExecutor.ExtractResultSetFromDbDataReaderAsync(reader); - - if (catalogRows.Rows.Count == 0) - { - return null; - } - - ObjectCatalogMetadata catalogMetadata = new(); - Dictionary nullabilityByColumn = new(StringComparer.OrdinalIgnoreCase); - Dictionary indexKeysByIndexId = new(); - - foreach (DbResultSetRow catalogRow in catalogRows.Rows) - { - Dictionary columnInfo = catalogRow.Columns; - - if (columnInfo["COLUMN_NAME"] is not string columnName) - { - continue; - } - - if (columnInfo["IS_HIDDEN"] is bool isHidden && isHidden) - { - catalogMetadata.HiddenColumns.Add(columnName); - } - - if (columnInfo["IS_IDENTITY"] is bool isIdentity && isIdentity) - { - catalogMetadata.IdentityColumns.Add(columnName); - } - - // A column the catalog does not describe as non-nullable is treated as nullable, so - // an unreadable flag can only disqualify a unique key, never promote one. - nullabilityByColumn[columnName] = columnInfo["IS_NULLABLE"] is not bool isNullable || isNullable; - - // A column outside every eligible unique index carries nulls for the index members. - if (columnInfo["INDEX_ID"] is not int indexId - || columnInfo["KEY_ORDINAL"] is not byte keyOrdinal) - { - continue; - } - - if (!indexKeysByIndexId.TryGetValue(indexId, out CatalogIndexKey? indexKey)) - { - indexKey = new CatalogIndexKey - { - IsPrimaryKey = columnInfo["IS_PRIMARY_KEY"] is bool isPrimaryKey && isPrimaryKey - }; - indexKeysByIndexId[indexId] = indexKey; - } - - indexKey.Columns.Add((columnName, keyOrdinal)); - } - - // Index order is creation order, which makes the candidate choice deterministic. - List indexIds = new(indexKeysByIndexId.Keys); - indexIds.Sort(); - - foreach (int indexId in indexIds) - { - CatalogIndexKey indexKey = indexKeysByIndexId[indexId]; - indexKey.Columns.Sort((left, right) => left.KeyOrdinal.CompareTo(right.KeyOrdinal)); - - List keyColumns = new(); - bool holdsNoNull = true; - - foreach ((string ColumnName, byte KeyOrdinal) keyColumn in indexKey.Columns) - { - keyColumns.Add(keyColumn.ColumnName); - - if (nullabilityByColumn.TryGetValue(keyColumn.ColumnName, out bool isNullable) && isNullable) - { - holdsNoNull = false; - } - } - - if (indexKey.IsPrimaryKey) - { - catalogMetadata.PrimaryKeyColumns.AddRange(keyColumns); - } - else if (holdsNoNull) - { - // Matches the data adapter, which promotes a unique key to the primary key only - // when none of its columns can hold a null. - catalogMetadata.UniqueKeyCandidates.Add(keyColumns); - } - } - - return catalogMetadata; - } - - /// - protected override async Task<(List Key, string? UnreachableKeyReason)> GetProjectionKeyFromResultSetAsync( - string selectStatement) - { - List projectionColumns = await DescribeProjectionAsync(selectStatement); - - if (projectionColumns.Count == 0) - { - return (new List(), null); - } - - // The key is taken from the underlying object's own catalog entry rather than from - // is_part_of_unique_key, which is reported per column as the union of every unique key - // of the result. That union cannot be split back into one key: a view over a table with - // both a primary key and a unique index would yield the two fused into an - // over-specified key, and a key member the projection left out comes back flagged - // hidden, which would yield a partial key that does not identify a row at all. - string? sourceSchema = null; - string? sourceTable = null; - Dictionary projectionBySourceColumn = new(StringComparer.OrdinalIgnoreCase); - HashSet sourceColumnsSeen = new(StringComparer.OrdinalIgnoreCase); - - foreach (ProjectionSourceColumn projectionColumn in projectionColumns) - { - if (projectionColumn.SourceSchema is null - || projectionColumn.SourceTable is null - || projectionColumn.SourceColumn is null) - { - continue; - } - - // Every row counts toward this check, the hidden ones included. Browse mode reports - // the key columns of each participating object, so an object whose every selected - // column was dropped from the projection survives in hidden rows alone. Skipping - // those first would read a join as a single object and infer one side's key for a - // result the other side can duplicate rows of. - if (sourceSchema is null) - { - sourceSchema = projectionColumn.SourceSchema; - sourceTable = projectionColumn.SourceTable; - } - else if (!string.Equals(sourceSchema, projectionColumn.SourceSchema, StringComparison.Ordinal) - || !string.Equals(sourceTable, projectionColumn.SourceTable, StringComparison.Ordinal)) - { - // More than one underlying object. Inferring a key across a join is out of - // scope; such an entity needs source.key-fields, as it did before this change. - return (new List(), null); - } - - // A source column reported twice means the result draws on more than one instance - // of the object: a self-join, which the check above cannot see, because browse mode - // reports the physical table rather than the alias. Such a join duplicates rows of - // the object just as any other does, so its key does not identify a row of the - // result. Declining is conservative — a projection that selects one column twice - // under different names is refused too, and simply needs source.key-fields, as it - // did before this change — and the alternative is a key that silently matches more - // than one row on an update or a delete. - if (!sourceColumnsSeen.Add(projectionColumn.SourceColumn)) - { - return (new List(), null); - } - - // Only a column the projection selects can carry a key: a hidden one is exactly - // what the projection left out. - if (!projectionColumn.IsHidden) - { - projectionBySourceColumn[projectionColumn.SourceColumn] = projectionColumn.Name; - } - } - - if (sourceSchema is null || sourceTable is null) - { - return (new List(), null); - } - - ObjectCatalogMetadata? sourceCatalogMetadata = - await GetCachedObjectCatalogMetadataAsync(sourceSchema, sourceTable); - - if (sourceCatalogMetadata is null) - { - return (new List(), null); - } - - // The underlying object's primary key comes first, then its unique keys, in index order - // — the preference the table path applies. Unlike a table, though, a view can simply - // not select the primary key, which is ordinary design rather than a key it cannot - // express, so a primary key that is out of reach does not end the search: a unique key - // the projection does carry whole identifies a row just as well. - List> sourceKeys = new(); - - if (sourceCatalogMetadata.PrimaryKeyColumns.Count > 0) - { - sourceKeys.Add(sourceCatalogMetadata.PrimaryKeyColumns); - } - - sourceKeys.AddRange(sourceCatalogMetadata.UniqueKeyCandidates); - - string? unreachableKeyColumn = null; - - foreach (List sourceKey in sourceKeys) - { - List projectionKey = new(); - string? missingKeyColumn = null; - - foreach (string sourceKeyColumn in sourceKey) - { - if (!projectionBySourceColumn.TryGetValue(sourceKeyColumn, out string? projectionColumnName)) - { - missingKeyColumn = sourceKeyColumn; - break; - } - - projectionKey.Add(projectionColumnName); - } - - if (missingKeyColumn is null && projectionKey.Count > 0) - { - return (projectionKey, null); - } - - // Remembered from the first key that came closest, to name something concrete if no - // key resolves at all. A partial key is never returned: it silently matches more - // than one row on an update or a delete. - unreachableKeyColumn ??= missingKeyColumn; - } - - if (unreachableKeyColumn is not null) - { - // Returned rather than thrown: an object whose source.key-fields is configured is - // reachable by that key, and rejecting it here would recommend exactly what was - // already configured. The caller reports this only if the object ends up keyless. - return ( - new List(), - $"No key of {sourceSchema}.{sourceTable}, which this object is built on, is fully exposed " - + $"by it: the nearest one includes the column {unreachableKeyColumn}, which this object does " - + "not expose. It therefore cannot be reached by key. Configure source.key-fields with columns " - + "it does expose that identify a row uniquely, if there are any."); - } - - return (new List(), null); - } - - /// - /// Describes a projection and returns what each of its columns resolves to in the catalog. - /// Browse information is what populates the source_* columns, and it is also what resolves a - /// view's columns to the object underneath. Metadata only: no row is read and no CLR type is - /// requested, so an unsupported column elsewhere in the object cannot fail this. - /// - private async Task> DescribeProjectionAsync(string selectStatement) - { - string statementParamName = $"{BaseQueryStructure.PARAM_NAME_PREFIX}param0"; - - string query = - "select r.name as COLUMN_NAME, r.source_schema as SOURCE_SCHEMA, " - + "r.source_table as SOURCE_TABLE, r.source_column as SOURCE_COLUMN, " - + "r.is_hidden as IS_HIDDEN " - + $"from sys.dm_exec_describe_first_result_set({statementParamName}, null, 1) as r " - + "order by r.column_ordinal;"; - - Dictionary parameters = new() - { - { statementParamName, new(selectStatement, DbType.String) } - }; - - try - { - List? projectionColumns = await QueryExecutor.ExecuteQueryAsync( - sqltext: query, - parameters: parameters, - dataReaderHandler: SummarizeProjectionSourceColumnsAsync, - dataSourceName: _dataSourceName); - - return projectionColumns ?? new List(); - } - catch (Exception ex) - { - // The statement may not be describable, or the dynamic management function may be - // unavailable, as on a dedicated SQL pool. Reporting nothing leaves the behavior to - // the missing-primary-key path, which is where it was before this change. - _logger.LogWarning( - "Unable to describe the projection of {selectStatement} to infer a key: {message}", - selectStatement, - ex.Message); - - return new List(); - } - } - - private async Task> SummarizeProjectionSourceColumnsAsync( - DbDataReader reader, - List? args = null) - { - DbResultSet resultSet = await QueryExecutor.ExtractResultSetFromDbDataReaderAsync(reader); - - List projectionColumns = new(); - - foreach (DbResultSetRow row in resultSet.Rows) - { - if (row.Columns["COLUMN_NAME"] is not string columnName) - { - continue; - } - - projectionColumns.Add(new ProjectionSourceColumn - { - Name = columnName, - SourceSchema = row.Columns["SOURCE_SCHEMA"] as string, - SourceTable = row.Columns["SOURCE_TABLE"] as string, - SourceColumn = row.Columns["SOURCE_COLUMN"] as string, - IsHidden = row.Columns["IS_HIDDEN"] is bool isHidden && isHidden - }); - } - - return projectionColumns; - } - - /// - /// One column of a described projection, with the catalog object it resolves to. - /// - private sealed class ProjectionSourceColumn - { - public string Name { get; init; } = string.Empty; - - public string? SourceSchema { get; init; } - - public string? SourceTable { get; init; } - - public string? SourceColumn { get; init; } - - public bool IsHidden { get; init; } - } - - /// - /// One unique index of a database object, while its key columns are being collected. - /// - private sealed class CatalogIndexKey - { - public bool IsPrimaryKey { get; init; } - - public List<(string ColumnName, byte KeyOrdinal)> Columns { get; } = new(); - } - public override string GetDefaultSchemaName() { return "dbo"; diff --git a/src/Core/Services/MetadataProviders/SqlMetadataProvider.cs b/src/Core/Services/MetadataProviders/SqlMetadataProvider.cs index 08e948f1fb..1fe5c32eb6 100644 --- a/src/Core/Services/MetadataProviders/SqlMetadataProvider.cs +++ b/src/Core/Services/MetadataProviders/SqlMetadataProvider.cs @@ -1,8 +1,6 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -using System.Collections.Concurrent; -using System.Collections.Immutable; using System.Collections.ObjectModel; using System.Data; using System.Data.Common; @@ -14,6 +12,7 @@ using System.Text.Json.Nodes; using Azure.DataApiBuilder.Config.DatabasePrimitives; using Azure.DataApiBuilder.Config.ObjectModel; +using Azure.DataApiBuilder.Core.Authorization; using Azure.DataApiBuilder.Core.Configurations; using Azure.DataApiBuilder.Core.Models; using Azure.DataApiBuilder.Core.Parsers; @@ -71,17 +70,6 @@ public abstract class SqlMetadataProvider : protected const int NUMBER_OF_RESTRICTIONS = 4; - /// - /// Column data types, as reported by the "Columns" schema collection, that the data - /// provider cannot map to a CLR type. Reading such a column makes - /// fail with - /// "DataReader.GetFieldType(N) returned null", which takes down the whole database object - /// even when the column itself is never exposed. They are therefore left out of the - /// projection used for schema discovery. - /// Empty by default: a provider only lists a type here when it genuinely cannot resolve it. - /// - protected virtual ImmutableHashSet UnsupportedColumnDataTypes => ImmutableHashSet.Empty; - protected string ConnectionString { get; init; } protected IQueryBuilder SqlQueryBuilder { get; init; } @@ -96,148 +84,8 @@ public abstract class SqlMetadataProvider : private Dictionary> EntityExposedNamesToBackingColumnNames { get; } = new(); - /// - /// Caches the "Columns" schema collection per database object for the duration of metadata - /// initialization, so schema discovery and column definition population share one catalog - /// round trip instead of querying twice per object. Cleared once initialization completes. - /// The key is compared with an ordinal comparer: under a case-sensitive collation - /// dbo.Foo and dbo.foo are distinct objects, and aliasing them would serve one - /// object's catalog rows for the other. Case-insensitive collations are unaffected, since - /// they cannot hold both names at once. - /// - private readonly ConcurrentDictionary _columnsMetadataCache = new(StringComparer.Ordinal); - - /// - /// Columns left out of the schema projection per database object, mapped to the data type - /// that made them unreadable. Used to explain the omission when a configured primary key - /// turns out to be one of them. Keyed by object with an ordinal comparer for the reason - /// above; the inner column names stay case-insensitive, matching how this class resolves - /// configured field names against the schema. - /// - private readonly ConcurrentDictionary> _skippedColumnsByObject = new(StringComparer.Ordinal); - - /// - /// Catalog facts per database object that the "Columns" schema collection does not report. - /// Only populated for providers that declare , since - /// only those replace "*" with an explicit projection and therefore need them. Keyed by - /// object with an ordinal comparer, for the reason given on . - /// - private readonly ConcurrentDictionary _objectCatalogMetadataCache = new(StringComparer.Ordinal); - - /// - /// Why no key of the object underneath a narrowed projection is reachable through it, for an - /// object the catalog holds no index for. Recorded rather than thrown where it is found, - /// because the schema read runs for every object while the answer only matters for one that - /// ends up with no key: an object whose source.key-fields is configured must not be rejected - /// over a key it was told not to use. Keyed by object with an ordinal comparer, for the - /// reason given on . - /// - private readonly ConcurrentDictionary _unreachableKeyReasonByObject = new(StringComparer.Ordinal); - - /// - /// The catalog facts an explicit schema projection needs and the "Columns" schema collection - /// does not carry: which columns the database hides from "SELECT *", which columns identify - /// a row, and which are identity columns. Together they replace what the data adapter - /// reports under CommandBehavior.KeyInfo, which cannot be used once the projection is - /// narrowed: the adapter appends key columns the projection left out as hidden reader - /// columns, and an unsupported type among them reintroduces the very failure the narrowing - /// avoids. - /// - protected sealed class ObjectCatalogMetadata - { - /// - /// Columns the database omits from "SELECT *" — for SQL Server, the period columns of a - /// temporal table declared GENERATED ALWAYS ... HIDDEN. Naming them in a projection - /// would expose columns that are invisible today, so they are subtracted before the - /// unsupported data types are. - /// - public HashSet HiddenColumns { get; } = new(StringComparer.OrdinalIgnoreCase); - - /// - /// Identity columns. Carried separately rather than through - /// , whose setter coerces a DataType it cannot - /// increment to Int32: SQL Server allows identity on tinyint, numeric and decimal, and - /// the coercion would report the wrong SystemType, which reaches parameter typing and - /// the generated API schemas. - /// - public HashSet IdentityColumns { get; } = new(StringComparer.OrdinalIgnoreCase); - - /// - /// The columns of the object's own primary key, in key order. Empty when the object has - /// none. - /// - public List PrimaryKeyColumns { get; } = new(); - - /// - /// Unique indexes eligible to identify a row when the object has no primary key, in - /// index order, each holding its key columns in key order. Only indexes whose every key - /// column is non-nullable qualify. - /// Preserves what the data adapter does on the unnarrowed path: absent a primary key it - /// reports such a unique key as , and dropping that - /// would make the narrowed path demand source.key-fields for an object the other path - /// resolves on its own. - /// - public List> UniqueKeyCandidates { get; } = new(); - } - - /// - /// Reads for a database object. Returns null by default: - /// a provider only implements this when it declares . - /// When it returns null the projection stays "*" and schema discovery behaves exactly as it - /// did before. - /// - protected virtual Task GetObjectCatalogMetadataAsync( - string schemaName, - string tableName) - { - return Task.FromResult(null); - } - - /// - /// Returns the columns of a unique key the given projection exposes, in key order, for an - /// object the catalog holds no index for — a view, whose key the data adapter resolved - /// through the underlying table. Empty by default, and empty whenever the provider cannot - /// answer, which leaves the missing-primary-key error to be reported as before. - /// A key column the projection left out must never be reported as part of a key: a partial - /// key silently matches more than one row on an update or a delete. - /// - /// - /// The key, and a reason when the provider can tell that no key of the underlying object is - /// fully exposed and can name what is missing. The reason is only reported when the object - /// ends up with no key at all: a configured source.key-fields settles the question, and - /// failing here anyway would reject an object whose configuration already says how to reach - /// it — while recommending exactly what was configured. - /// - protected virtual Task<(List Key, string? UnreachableKeyReason)> GetProjectionKeyFromResultSetAsync( - string selectStatement) - { - return Task.FromResult<(List, string?)>((new List(), null)); - } - - /// - /// Returns for a database object, reading the catalog - /// once per object for the duration of metadata initialization. - /// - protected async Task GetCachedObjectCatalogMetadataAsync( - string schemaName, - string tableName) - { - string cacheKey = GetObjectCacheKey(schemaName, tableName); - - if (_objectCatalogMetadataCache.TryGetValue(cacheKey, out ObjectCatalogMetadata? cachedMetadata)) - { - return cachedMetadata; - } - - ObjectCatalogMetadata? catalogMetadata = await GetObjectCatalogMetadataAsync(schemaName, tableName); - - if (catalogMetadata is not null) - { - _objectCatalogMetadataCache[cacheKey] = catalogMetadata; - } - - return catalogMetadata; - } + // Keyed by schema table, which entities over the same database object share. + private readonly Dictionary _unreadableColumnsBySchemaTable = new(); protected IAbstractQueryManagerFactory QueryManagerFactory { get; init; } @@ -503,18 +351,10 @@ public async Task InitializeAsync(CancellationToken cancellationToken) _runtimeConfigValidator.ValidateEntityAndAutoentityConfigurations(runtimeConfig); GenerateDatabaseObjectForEntities(); - - try - { - await PopulateObjectDefinitionForEntities(cancellationToken); - } - finally - { - ReleaseCatalogMetadataCaches(); - } - + await PopulateObjectDefinitionForEntities(cancellationToken); cancellationToken.ThrowIfCancellationRequested(); GenerateExposedToBackingColumnMapsForEntities(); + ValidateReferencesToUnreadableColumns(); // When IsLateConfigured is true we are in a hosted scenario and do not reveal primary key information. if (!_runtimeConfigProvider.IsLateConfigured) @@ -1452,17 +1292,6 @@ await PopulateObjectDefinitionForEntity( cancellationToken); } - try - { - // After the loops above, not inside them: a relationship's "target.fields" belong - // to the other entity's object, which may not have been read yet while this one is. - RejectConfiguredRelationshipReferencesToSkippedColumns(); - } - catch (Exception e) - { - HandleOrRecordException(e); - } - try { await PopulateForeignKeyDefinitionAsync(cancellationToken); @@ -1789,9 +1618,16 @@ private async Task PopulateSourceDefinitionAsync( if (sourceDefinition.PrimaryKey.Count == 0) { - // When the object's own primary key is unreadable, say so. The message below reads - // as a configuration mistake, and no configuration can express that key. - RejectUnreadablePrimaryKey(schemaName, tableName); + UnreadableColumns? unreadableColumns = GetUnreadableColumns(schemaName, tableName); + if (unreadableColumns is not null && unreadableColumns.KeyColumns.Count > 0) + { + throw new DataApiBuilderException( + message: $"Entity '{entityName}' ({schemaName}.{tableName}) has no usable primary key: key column(s) " + + $"{FormatColumns(unreadableColumns.TypeNameByColumn.Where(column => unreadableColumns.KeyColumns.Contains(column.Key)))} " + + "have a data type that is not supported. Configure source.key-fields with supported columns that identify a row.", + statusCode: HttpStatusCode.ServiceUnavailable, + subStatusCode: DataApiBuilderException.SubStatusCodes.ErrorInInitialization); + } throw new DataApiBuilderException( message: $"Primary key not configured on the given database object {tableName}", @@ -1815,6 +1651,12 @@ await PopulateTriggerMetadataForTable( schemaName, tableName, cancellationToken); + + if (_unreadableColumnsBySchemaTable.TryGetValue(dataTable, out UnreadableColumns? skippedColumns)) + { + ThrowIfReferencesUnreadableColumn(entityName, skippedColumns, sourceDefinition.PrimaryKey, "the primary key"); + } + using DataTableReader reader = new(dataTable); DataTable schemaTable = reader.GetSchemaTable(); RuntimeConfig runtimeConfig = _runtimeConfigProvider.GetConfig(); @@ -1859,13 +1701,10 @@ await PopulateTriggerMetadataForTable( sourceDefinition.Columns.TryAdd(columnName, column); } - ApplyIdentityColumnsFromCatalog(schemaName, tableName, sourceDefinition); - - RejectPrimaryKeyOnUnsupportedColumn(schemaName, tableName, sourceDefinition); - - RejectConfiguredReferencesToSkippedColumns(entityName, entity, schemaName, tableName); - - DataTable columnsInTable = await GetCachedColumnsAsync(schemaName, tableName, cancellationToken); + DataTable columnsInTable = await GetColumnsAsync( + schemaName, + tableName, + cancellationToken); PopulateColumnDefinitionWithHasDefaultAndDbType( sourceDefinition, @@ -1992,6 +1831,7 @@ private async Task GetTableWithSchemaFromDataSetAsync( try { dataTable = await FillSchemaForTableAsync( + entityName, schemaName, tableName, cancellationToken); @@ -2074,11 +1914,9 @@ await QueryExecutor.SetManagedIdentityAccessTokenIfAnyAsync( /// /// Using a data adapter, obtains the schema of the given table name /// and adds the corresponding DataTable to the entities data set. - /// Columns whose data type the data provider cannot map to a CLR type are left out of the - /// projection, because the data adapter refuses to build a schema mapping for them and the - /// whole object would otherwise be unreachable. See . /// private async Task FillSchemaForTableAsync( + string entityName, string schemaName, string tableName, CancellationToken cancellationToken) @@ -2125,77 +1963,17 @@ await QueryExecutor.SetManagedIdentityAccessTokenIfAnyAsync( innerException: ex); } - string tableNameWithSchemaPrefix = GetTableNameWithSchemaPrefix(schemaName, tableName); - - // Resolved before the connection below is opened. Reading the catalog uses a connection - // of its own, and nesting that inside an already open one exhausts a small pool: with - // "Max Pool Size=1" the inner open waits for a connection the outer scope still holds. - string projection = await BuildSchemaProjectionAsync(schemaName, tableName, cancellationToken); - - bool isProjectionNarrowed = !string.Equals(projection, "*", StringComparison.Ordinal); - - // Resolved before the connection below is opened, for the same reason as the projection: - // reading the catalog uses a connection of its own, and nesting that inside an already - // open one exhausts a small pool. - ObjectCatalogMetadata? catalogMetadata = isProjectionNarrowed - ? await GetCachedObjectCatalogMetadataAsync(schemaName, tableName) - : null; - - string selectStatement = $"SELECT {projection} FROM {tableNameWithSchemaPrefix}"; - - // An ordinary view has no indexes of its own, so the catalog lookup above finds no key - // for it. FillSchema resolved one through the view's underlying table under KeyInfo, and - // describing the projection recovers the same route — without a reader, and without - // asking for a CLR type. Only reached for an object the catalog could not key, and - // resolved before the connection is opened for the same pooling reason as above. - // Kept out of the cached metadata: it depends on this projection, not on the object. - List describedKey = new(); - - if (catalogMetadata is not null - && catalogMetadata.PrimaryKeyColumns.Count == 0 - && catalogMetadata.UniqueKeyCandidates.Count == 0) - { - (List Key, string? UnreachableKeyReason) described = - await GetProjectionKeyFromResultSetAsync(selectStatement); - - describedKey = described.Key; - - if (described.UnreachableKeyReason is not null) - { - _unreachableKeyReasonByObject[GetObjectCacheKey(schemaName, tableName)] = - described.UnreachableKeyReason; - } - } - await conn.OpenAsync(cancellationToken); - if (isProjectionNarrowed) - { - // The projection left columns out, so the data adapter cannot be used here: - // FillSchema runs with CommandBehavior.KeyInfo, under which the provider performs - // its own key discovery and appends key columns missing from the SELECT list as - // hidden reader columns. A column whose CLR type the provider cannot resolve - // reintroduces "DataReader.GetFieldType(N) returned null" that way even though the - // projection excluded it — reachable through the object's own primary key, and - // through any unique index, including when a supported key is configured through - // source.key-fields. Reading the shape without KeyInfo keeps the projection - // authoritative; the primary key comes from the catalog instead. - return await ReadSchemaWithoutKeyInfoAsync( - conn, - selectStatement, - tableNameWithSchemaPrefix, - schemaName, - tableName, - catalogMetadata, - describedKey); - } - using DataAdapterT adapterForTable = new(); using CommandT selectCommand = new() { - Connection = conn, - CommandText = selectStatement + Connection = conn }; + + string tableNameWithSchemaPrefix = GetTableNameWithSchemaPrefix(schemaName, tableName); + selectCommand.CommandText + = $"SELECT * FROM {tableNameWithSchemaPrefix}"; adapterForTable.SelectCommand = selectCommand; cancellationToken.ThrowIfCancellationRequested(); @@ -2230,734 +2008,359 @@ await QueryExecutor.SetManagedIdentityAccessTokenIfAnyAsync( ex, cancellationToken); } - - cancellationToken.ThrowIfCancellationRequested(); - return dataTable[0]; - } - - /// - /// Reads the shape of a narrowed projection without CommandBehavior.KeyInfo and registers - /// the result in under the name the data adapter would have - /// used, so callers find it there on subsequent lookups. The primary key is taken from the - /// catalog, because the adapter's own key discovery is precisely what has to be avoided. - /// - private async Task ReadSchemaWithoutKeyInfoAsync( - ConnectionT conn, - string selectStatement, - string tableNameWithSchemaPrefix, - string schemaName, - string tableName, - ObjectCatalogMetadata? catalogMetadata, - List describedKey) - { - DataTable dataTable = new(tableNameWithSchemaPrefix); - - using (CommandT selectCommand = new()) + catch (InvalidOperationException) { - selectCommand.Connection = conn; - selectCommand.CommandText = selectStatement; - - // SchemaOnly describes the statement without returning rows. Without KeyInfo the - // reader carries exactly the projected columns and nothing else. - using DbDataReader reader = - await selectCommand.ExecuteReaderAsync(CommandBehavior.SchemaOnly); - - using DataTable? schemaTable = reader.GetSchemaTable(); - - if (schemaTable is null) - { - throw new DataApiBuilderException( - message: $"The data provider reported no schema for {schemaName}.{tableName}.", - statusCode: HttpStatusCode.ServiceUnavailable, - subStatusCode: DataApiBuilderException.SubStatusCodes.ErrorInInitialization); - } + // The data adapter rejects the whole object when the provider reports no CLR type for a column. + DataTable? schemaWithoutUnreadableColumns = await FillSchemaWithoutUnreadableColumnsAsync( + entityName, + schemaName, + tableName, + selectCommand, + cancellationToken); - foreach (DataRow columnInfo in schemaTable.Rows) + if (schemaWithoutUnreadableColumns is null) { - if (columnInfo["ColumnName"] is not string columnName) - { - continue; - } - - // The provider-reported DataType is carried through unchanged. Auto-increment is - // deliberately not set here — see ApplyIdentityColumnsFromCatalog for why - // DataColumn.AutoIncrement cannot be the transport for it. - DataColumn column = new(columnName, (Type)columnInfo["DataType"]) - { - // Unknown nullability is treated as nullable. A column wrongly marked - // non-nullable is reported as required through REST, GraphQL and OpenAPI and - // rejects writes the database would accept, so the permissive direction is - // the safe one when the provider does not report the flag. - AllowDBNull = columnInfo["AllowDBNull"] is not bool allowDbNull || allowDbNull - }; - - dataTable.Columns.Add(column); + throw; } - } - - if (catalogMetadata is not null) - { - DataColumn[]? keyColumns = ResolveKeyColumns(dataTable, catalogMetadata) - ?? (describedKey.Count > 0 ? TryResolveColumns(dataTable, describedKey) : null); - if (keyColumns is not null) - { - dataTable.PrimaryKey = keyColumns; - } + return schemaWithoutUnreadableColumns; } - EntitiesDataSet.Tables.Add(dataTable); - - return dataTable; + cancellationToken.ThrowIfCancellationRequested(); + return dataTable[0]; } /// - /// Picks the columns to report as on the narrowed path. - /// The object's own primary key wins. A key column the projection left out is not reported, - /// and the key is not silently replaced by another candidate either: - /// PopulateSourceDefinitionAsync fails through RejectUnreadablePrimaryKey when that key is - /// the one in effect, and a key configured through source.key-fields takes precedence over - /// this one anyway. - /// Absent a primary key, the first unique index whose every key column is present is used, - /// which is what the data adapter reports on the unnarrowed path. Returns null when nothing - /// identifies a row, leaving the caller to report a missing primary key as it does today. + /// Builds the schema the data adapter would have built, minus the columns the provider reports no CLR + /// type for, such as SQL Server's geometry, geography and hierarchyid. Returns null when there are none, + /// so the caller rethrows the adapter's error. /// - private static DataColumn[]? ResolveKeyColumns(DataTable dataTable, ObjectCatalogMetadata catalogMetadata) + private async Task FillSchemaWithoutUnreadableColumnsAsync( + string entityName, + string schemaName, + string tableName, + DbCommand selectCommand, + CancellationToken cancellationToken) { - if (catalogMetadata.PrimaryKeyColumns.Count > 0) - { - return TryResolveColumns(dataTable, catalogMetadata.PrimaryKeyColumns); - } - - foreach (List uniqueKeyCandidate in catalogMetadata.UniqueKeyCandidates) + DataTable? readerSchema; + try { - DataColumn[]? keyColumns = TryResolveColumns(dataTable, uniqueKeyCandidate); - - if (keyColumns is not null) - { - return keyColumns; - } + using DbDataReader reader = await selectCommand.ExecuteReaderAsync( + CommandBehavior.SchemaOnly | CommandBehavior.KeyInfo, + cancellationToken); + readerSchema = await reader.GetSchemaTableAsync(cancellationToken); } - - return null; - } - - /// - /// Resolves every named column against the table, in the order given, or returns null when - /// one of them is absent. - /// - private static DataColumn[]? TryResolveColumns(DataTable dataTable, List columnNames) - { - DataColumn[] columns = new DataColumn[columnNames.Count]; - - for (int index = 0; index < columnNames.Count; index++) + catch (Exception ex) when (cancellationToken.IsCancellationRequested) { - if (!dataTable.Columns.Contains(columnNames[index])) - { - return null; - } - - columns[index] = dataTable.Columns[columnNames[index]]!; + throw new OperationCanceledException( + "Schema discovery was canceled during shutdown.", + ex, + cancellationToken); } - return columns; - } + DataRow[] columns = readerSchema?.Rows.Cast() + .OrderBy(column => Convert.ToInt32(column[SchemaTableColumn.ColumnOrdinal], System.Globalization.CultureInfo.InvariantCulture)) + .ToArray() ?? []; - /// - /// Builds the projection used to read the schema of a database object. Returns "*" unless - /// the object holds columns whose data type this provider cannot map to a CLR type, in - /// which case those columns are named out of the projection so the rest stays reachable. - /// The column list comes from the "Columns" schema collection, which reads catalog metadata - /// only and therefore never has to materialize the offending type. - /// - /// - /// Thrown when every column of the object has an unsupported data type. Returning "*" there - /// would re-issue the projection that cannot be read, hiding the reason behind the - /// provider's own error. - /// - private async Task BuildSchemaProjectionAsync( - string schemaName, - string tableName, - CancellationToken cancellationToken) - { - if (UnsupportedColumnDataTypes.Count == 0) + if (!columns.Any(column => column[SchemaTableColumn.DataType] is not Type)) { - return "*"; + return null; } - List readableColumns = new(); - Dictionary skippedColumns = new(StringComparer.OrdinalIgnoreCase); + DataTable schemaTable = new(GetTableNameWithSchemaPrefix(schemaName, tableName)); + Dictionary unreadableColumns = new(StringComparer.OrdinalIgnoreCase); - try + foreach (DataRow column in columns.Where(column => !IsSchemaFlagSet(column, SchemaTableOptionalColumn.IsHidden))) { - DataTable columnsInTable = await GetCachedColumnsAsync(schemaName, tableName, cancellationToken); - - // Classify from the "Columns" schema collection first, which is read for every - // object anyway. Only an object that actually holds an unsupported type needs the - // additional catalog facts below; asking for them up front would add a query per - // MSSQL and DWSQL object to every startup. - List supportedColumns = new(); - - foreach (DataRow columnInfo in columnsInTable.Rows) - { - if (columnInfo["COLUMN_NAME"] is not string columnName) - { - continue; - } - - if (columnInfo["DATA_TYPE"] is not string dataType) - { - // The catalog did not report a usable type name, so this column cannot be - // classified. Leave the projection alone rather than guess. - return "*"; - } - - if (UnsupportedColumnDataTypes.Contains(dataType)) - { - skippedColumns[columnName] = dataType; - } - else - { - supportedColumns.Add(columnName); - } - } - - if (skippedColumns.Count == 0) - { - return "*"; - } - - if (supportedColumns.Count == 0) - { - // Falling back to "*" here would re-issue the very projection that fails, so the - // caller would see the opaque provider error instead of the reason for it. - throw new DataApiBuilderException( - message: $"Every column of {schemaName}.{tableName} has a data type that is not supported: " - + $"{FormatSkippedColumns(skippedColumns)}. The object cannot be exposed.", - statusCode: HttpStatusCode.ServiceUnavailable, - subStatusCode: DataApiBuilderException.SubStatusCodes.ErrorInInitialization); - } - - ObjectCatalogMetadata? catalogMetadata = - await GetCachedObjectCatalogMetadataAsync(schemaName, tableName); - - if (catalogMetadata is null) + string columnName = (string)column[SchemaTableColumn.ColumnName]; + if (column[SchemaTableColumn.DataType] is Type dataType) { - // Without the catalog there is no way to tell which columns the database hides - // from "SELECT *". Naming columns anyway would add the hidden period columns of - // a temporal table to the exposed contract, so leave the projection alone. - return "*"; + schemaTable.Columns.Add(CreateColumnAsDataAdapterWould(column, columnName, dataType)); } - - foreach (string columnName in supportedColumns) + else { - if (catalogMetadata.HiddenColumns.Contains(columnName)) - { - // "SELECT *" does not return this column, so naming it would widen the - // exposed contract instead of preserving it. It is not "skipped": it was - // never part of the object's shape as the engine sees it, and the read-only - // classification does not recognize generated-always period columns, so a - // PUT would try to null them. - continue; - } - - readableColumns.Add(columnName); + unreadableColumns[columnName] = GetDatabaseTypeName(column); } } - catch (Exception ex) when (ex is not DataApiBuilderException) - { - // The column list is a best-effort optimization: without it the read below behaves - // exactly as it did before, failing loudly if an unsupported type is present. - _logger.LogDebug( - "Unable to enumerate the columns of {schemaName}.{tableName}: {message}", - schemaName, - tableName, - ex.Message); - return "*"; - } - if (readableColumns.Count == 0) + if (schemaTable.Columns.Count == 0) { - // Every column that is not of an unsupported type is one the database hides from - // "SELECT *". Naming the hidden ones is not an option, and "*" would re-issue the - // projection that fails, so nothing about this object can be read. throw new DataApiBuilderException( - message: $"No column of {schemaName}.{tableName} can be read: " - + $"{FormatSkippedColumns(skippedColumns)} have a data type that is not supported, and " - + "every remaining column is one the database does not return from a SELECT *. " - + "The object cannot be exposed.", + message: $"Entity '{entityName}' ({schemaName}.{tableName}) cannot be exposed: every column has a data type " + + $"that is not supported: {FormatColumns(unreadableColumns)}.", statusCode: HttpStatusCode.ServiceUnavailable, subStatusCode: DataApiBuilderException.SubStatusCodes.ErrorInInitialization); } - _skippedColumnsByObject[GetObjectCacheKey(schemaName, tableName)] = skippedColumns; - - _logger.LogWarning( - "Skipping column(s) of {schemaName}.{tableName} whose data type is not supported: {skippedColumns}. " - + "They are not exposed through REST, GraphQL or MCP.", - schemaName, - tableName, - FormatSkippedColumns(skippedColumns)); + List keyRows = GetKeyAsDataAdapterWould(columns); + List unreadableKeyColumns = keyRows + .Where(column => column[SchemaTableColumn.DataType] is not Type) + .Select(column => (string)column[SchemaTableColumn.ColumnName]) + .ToList(); - return string.Join(", ", readableColumns.Select(column => SqlQueryBuilder.QuoteIdentifier(column))); - } - - /// - /// Fails initialization when a configured primary key names a column that was left out of - /// the projection because its data type is not supported. The key would otherwise stay in - /// while being absent from - /// , and the inconsistency surfaces much later as a - /// lookup failure while building queries, the OpenAPI document or the EDM model. - /// - private void RejectPrimaryKeyOnUnsupportedColumn( - string schemaName, - string tableName, - SourceDefinition sourceDefinition) - { - if (!_skippedColumnsByObject.TryGetValue(GetObjectCacheKey(schemaName, tableName), out Dictionary? skippedColumns)) + if (keyRows.Count > 0 && unreadableKeyColumns.Count == 0) { - return; - } + DataColumn[] keyColumns = keyRows + .Select(column => schemaTable.Columns[(string)column[SchemaTableColumn.ColumnName]]!) + .ToArray(); - foreach (string primaryKey in sourceDefinition.PrimaryKey) - { - if (skippedColumns.TryGetValue(primaryKey, out string? dataType)) + // Like the data adapter, only a key without nullable columns becomes the primary key. + if (keyColumns.All(keyColumn => !keyColumn.AllowDBNull)) { - throw new DataApiBuilderException( - message: $"The primary key column {primaryKey} of {schemaName}.{tableName} has the data type " - + $"{dataType}, which is not supported. A primary key cannot be omitted from the object " - + "metadata, so this object cannot be exposed.", - statusCode: HttpStatusCode.ServiceUnavailable, - subStatusCode: DataApiBuilderException.SubStatusCodes.ErrorInInitialization); + schemaTable.PrimaryKey = keyColumns; } } - } - /// - /// Marks identity columns on the narrowed schema discovery path, which builds its columns - /// itself instead of letting the data adapter populate them. - /// The flag is not carried through : its setter - /// coerces a DataType it cannot increment to Int32, and SQL Server allows identity on - /// tinyint, numeric and decimal, so doing that would report the wrong SystemType and reach - /// parameter typing and the generated API schemas. It comes from the catalog instead. - /// No-op for every object whose projection was not narrowed, which is where the adapter - /// still reports the flag itself. - /// - private void ApplyIdentityColumnsFromCatalog( - string schemaName, - string tableName, - SourceDefinition sourceDefinition) - { - if (!_objectCatalogMetadataCache.TryGetValue( - GetObjectCacheKey(schemaName, tableName), - out ObjectCatalogMetadata? catalogMetadata)) - { - return; - } + EntitiesDataSet.Tables.Add(schemaTable); + _unreadableColumnsBySchemaTable[schemaTable] = new UnreadableColumns(unreadableColumns, unreadableKeyColumns); - foreach (string identityColumn in catalogMetadata.IdentityColumns) + if (unreadableColumns.Count > 0) { - if (sourceDefinition.Columns.TryGetValue(identityColumn, out ColumnDefinition? columnDefinition)) + // Hosted deployments don't log schema details; see LogPrimaryKeys. + if (_runtimeConfigProvider.IsLateConfigured) { - columnDefinition.IsAutoGenerated = true; - - // Matches how the adapter-reported flag is treated above: an auto-increment - // column is also read-only. - columnDefinition.IsReadOnly = true; + _logger.LogWarning( + "Skipped {columnCount} column(s) whose data type is not supported. They are not exposed through REST, GraphQL or MCP.", + unreadableColumns.Count); } - } - } - - /// - /// Fails initialization with the reason when the object's own primary key includes a column - /// left out of the projection because its data type is not supported. Without this the - /// caller reports a missing primary key, which reads as something the user forgot to - /// configure even though no configuration can express that key. - /// - private void RejectUnreadablePrimaryKey(string schemaName, string tableName) - { - string cacheKey = GetObjectCacheKey(schemaName, tableName); - - // Reached only with no key in effect, which is what makes this the right place to report - // it: the schema read found that no key of the object underneath is reachable through - // this projection, and nothing configured has since supplied one. - if (_unreachableKeyReasonByObject.TryGetValue(cacheKey, out string? unreachableKeyReason)) - { - throw new DataApiBuilderException( - message: unreachableKeyReason, - statusCode: HttpStatusCode.ServiceUnavailable, - subStatusCode: DataApiBuilderException.SubStatusCodes.ErrorInInitialization); - } - - if (!_skippedColumnsByObject.TryGetValue(cacheKey, out Dictionary? skippedColumns) - || !_objectCatalogMetadataCache.TryGetValue(cacheKey, out ObjectCatalogMetadata? catalogMetadata)) - { - return; - } - - foreach (string primaryKeyColumn in catalogMetadata.PrimaryKeyColumns) - { - if (skippedColumns.TryGetValue(primaryKeyColumn, out string? dataType)) + else { - throw new DataApiBuilderException( - message: $"The primary key of {schemaName}.{tableName} includes the column {primaryKeyColumn}, " - + $"whose data type {dataType} is not supported. The object cannot be exposed through that " - + "key. Configure source.key-fields with a supported column that identifies a row uniquely, " - + "if the object has one.", - statusCode: HttpStatusCode.ServiceUnavailable, - subStatusCode: DataApiBuilderException.SubStatusCodes.ErrorInInitialization); + _logger.LogWarning( + "Skipped column(s) of {databaseObject} whose data type is not supported: {columns}. They are not exposed through REST, GraphQL or MCP.", + $"{schemaName}.{tableName}", + FormatColumns(unreadableColumns)); } } + + return schemaTable; } /// - /// Fails initialization when the configuration names a column that was left out of the - /// projection because its data type is not supported. - /// Such a name keeps resolving after the column is gone: the exposed and backing column maps - /// are built from entity fields and mappings without requiring the column to exist in - /// , and the authorization resolver accepts explicitly - /// included field names. The reference then reaches code that indexes - /// and fails per request instead of at startup — a - /// permission-derived default projection selects the column and the read fails serializing - /// it, a mutation throws while resolving the backing column, and MCP metadata advertises a - /// field no other surface has. Rejecting here keeps the configuration and the exposed - /// contract in agreement. Nothing that worked before stops working: an object with such a - /// column failed discovery outright, so the entity never loaded. + /// Returns the key columns the data adapter would infer: the visible key columns, unless a hidden key + /// column of the same object shows that the key is not fully selected. /// - private void RejectConfiguredReferencesToSkippedColumns( - string entityName, - Entity? entity, - string schemaName, - string tableName) + private static List GetKeyAsDataAdapterWould(IEnumerable columns) { - if (entity is null - || !_skippedColumnsByObject.TryGetValue( - GetObjectCacheKey(schemaName, tableName), - out Dictionary? skippedColumnsForObject)) - { - return; - } - - // Held in a non-nullable local because the local function below captures it, and the - // guard above is not something the compiler can carry into that capture. - Dictionary skippedColumns = skippedColumnsForObject; + List keyColumns = new(); + string? keyBaseTable = null; + bool keySpansBaseTables = false; - // "mappings" and "fields" both key on the backing column name. - if (entity.Mappings is not null) + foreach (DataRow column in columns) { - foreach (string backingColumn in entity.Mappings.Keys) + if (!IsSchemaFlagSet(column, SchemaTableColumn.IsKey)) { - RejectReference(backingColumn, "mappings"); + continue; } - } - if (entity.Fields is not null) - { - foreach (FieldMetadata field in entity.Fields) - { - RejectReference(field.Name, "fields"); - } - } + string? baseTable = column[SchemaTableColumn.BaseTableName] as string; - foreach (EntityPermission permission in entity.Permissions) - { - foreach (EntityAction action in permission.Actions) + if (IsSchemaFlagSet(column, SchemaTableOptionalColumn.IsHidden)) { - // A database policy is parsed per request against the OData model, which is - // built from SourceDefinition.Columns. A policy naming a column that is not - // there fails every request for that role instead of the configuration being - // rejected once, at startup. - foreach (string policyField in EnumeratePolicyFieldReferences(action.Policy?.Database)) + if (keySpansBaseTables || baseTable == keyBaseTable) { - // Policy identifiers are exposed names, the way the OData model's properties - // are, so they are resolved through the configured aliases before being - // compared against backing column names. Without that, an alias over a - // supported column that happens to carry the name of a skipped one — a - // "Location" alias of a text column beside a skipped "Location" spatial - // column — would be rejected even though the policy references the - // supported field. - RejectReference( - ResolveBackingColumnName(entity, policyField), - $"database policy of role {permission.Role}"); + return new List(); } - if (action.Fields?.Include is null) + continue; + } + + if (baseTable != keyBaseTable) + { + if (keyBaseTable is null) { - continue; + keyBaseTable = baseTable; } - - // An include list names columns that have to be readable. An exclude list naming - // one of these columns asks for what already happened, so it is left alone. - foreach (string includedField in action.Fields.Include) + else { - RejectReference(includedField, $"permissions of role {permission.Role}"); + keySpansBaseTables = true; } } - } - - void RejectReference(string configuredName, string configurationSection) - { - if (!skippedColumns.TryGetValue(configuredName, out string? dataType)) - { - return; - } - throw new DataApiBuilderException( - message: $"The {configurationSection} of entity {entityName} reference the column {configuredName} " - + $"of {schemaName}.{tableName}, whose data type {dataType} is not supported. That column is not " - + "part of the exposed contract, so the reference cannot be honored. Remove it from the " - + "configuration.", - statusCode: HttpStatusCode.ServiceUnavailable, - subStatusCode: DataApiBuilderException.SubStatusCodes.ErrorInInitialization); + keyColumns.Add(column); } + + return keyColumns; } /// - /// Yields the field names a database policy references through its "@item." prefix. - /// Single-quoted literals are skipped, with a doubled quote read as an escaped quote inside - /// one, so a literal that merely contains the prefix is not mistaken for a reference. - /// Scanned by hand rather than parsed: the OData model the parser needs does not exist yet - /// at this point in initialization. The names yielded are exposed names, which the caller - /// resolves through the configured aliases. + /// Creates a column with the nullability and auto-increment flags the data adapter derives from a schema row. /// - private static IEnumerable EnumeratePolicyFieldReferences(string? databasePolicy) + private static DataColumn CreateColumnAsDataAdapterWould(DataRow column, string columnName, Type dataType) { - const string POLICY_FIELD_PREFIX = "@item."; + bool isKey = IsSchemaFlagSet(column, SchemaTableColumn.IsKey); + bool isAutoIncrement = IsSchemaFlagSet(column, SchemaTableOptionalColumn.IsAutoIncrement) && IsAutoIncrementType(dataType); + bool isUniqueNonKey = IsSchemaFlagSet(column, SchemaTableColumn.IsUnique) && !isKey && !dataType.IsArray; + bool isReadOnly = IsSchemaFlagSet(column, SchemaTableOptionalColumn.IsReadOnly); + bool allowsNull = !column.Table.Columns.Contains(SchemaTableColumn.AllowDBNull) + || column[SchemaTableColumn.AllowDBNull] is not bool allowDbNull + || allowDbNull; - if (string.IsNullOrWhiteSpace(databasePolicy)) + return new DataColumn(columnName, dataType) { - yield break; - } + AutoIncrement = isAutoIncrement, + AllowDBNull = allowsNull || !(isAutoIncrement || !isReadOnly || isKey || isUniqueNonKey) + }; + } - int index = 0; + // Mirrors the internal DataColumn.IsAutoIncrementType; the data adapter marks only these types auto-increment. + private static bool IsAutoIncrementType(Type dataType) => + dataType == typeof(int) || dataType == typeof(long) || dataType == typeof(short) || dataType == typeof(decimal) + || dataType == typeof(System.Numerics.BigInteger) || dataType == typeof(System.Data.SqlTypes.SqlInt32) + || dataType == typeof(System.Data.SqlTypes.SqlInt64) || dataType == typeof(System.Data.SqlTypes.SqlInt16) + || dataType == typeof(System.Data.SqlTypes.SqlDecimal); - while (index < databasePolicy.Length) - { - if (databasePolicy[index] == '\'') - { - index++; + private static bool IsSchemaFlagSet(DataRow column, string flag) => + column.Table.Columns.Contains(flag) && column[flag] is bool isSet && isSet; - while (index < databasePolicy.Length) - { - if (databasePolicy[index] != '\'') - { - index++; - continue; - } - - // A doubled quote is an escaped quote within the literal, not its end. - if (index + 1 < databasePolicy.Length && databasePolicy[index + 1] == '\'') - { - index += 2; - continue; - } + // SQL Server qualifies a CLR type's name with its database and schema, e.g. "master.sys.geometry". + private static string GetDatabaseTypeName(DataRow column) + { + string? typeName = column.Table.Columns.Contains("DataTypeName") ? column["DataTypeName"] as string : null; + return string.IsNullOrWhiteSpace(typeName) ? "unknown" : typeName[(typeName.LastIndexOf('.') + 1)..]; + } - index++; - break; - } + private static string FormatColumns(IEnumerable> typeNameByColumn) => + string.Join(", ", typeNameByColumn.Select(column => $"{column.Key} ({column.Value})")); - continue; - } + private UnreadableColumns? GetUnreadableColumns(string schemaName, string tableName) + { + DataTable? schemaTable = EntitiesDataSet.Tables[GetTableNameWithSchemaPrefix(schemaName, tableName)]; + return schemaTable is not null && _unreadableColumnsBySchemaTable.TryGetValue(schemaTable, out UnreadableColumns? columns) + ? columns + : null; + } - if (string.Compare( - databasePolicy, - index, - POLICY_FIELD_PREFIX, - 0, - POLICY_FIELD_PREFIX.Length, - StringComparison.OrdinalIgnoreCase) != 0) - { - index++; - continue; - } + private UnreadableColumns? GetUnreadableColumnsForEntity(string entityName) => + EntityToDatabaseObject.TryGetValue(entityName, out DatabaseObject? databaseObject) + && databaseObject is not DatabaseStoredProcedure + ? GetUnreadableColumns(databaseObject.SchemaName, databaseObject.Name) + : null; - int fieldStart = index + POLICY_FIELD_PREFIX.Length; - int fieldEnd = fieldStart; + /// + /// Fails initialization when configuration names a column that schema discovery skipped, instead of + /// letting every request that uses the reference fail. + /// + private void ValidateReferencesToUnreadableColumns() + { + if (_unreadableColumnsBySchemaTable.Count == 0) + { + return; + } - while (fieldEnd < databasePolicy.Length - && (char.IsLetterOrDigit(databasePolicy[fieldEnd]) || databasePolicy[fieldEnd] == '_')) + foreach ((string entityName, Entity entity) in Entities) + { + try { - fieldEnd++; + ValidateReferencesToUnreadableColumns(entityName, entity); } - - if (fieldEnd > fieldStart) + catch (Exception e) { - yield return databasePolicy[fieldStart..fieldEnd]; + HandleOrRecordException(e); } - - index = fieldEnd > fieldStart ? fieldEnd : fieldStart; } } - /// - /// Fails initialization when a configured relationship names a column that was left out of - /// a projection because its data type is not supported. Runs after every object definition - /// is populated, unlike the other reference checks: a relationship's "target.fields" belong - /// to the other entity's object, which may not have been read yet while this one is. - /// The reference would otherwise keep resolving — the exposed and backing column maps are - /// built from the configuration rather than from - /// — and fail per request instead, in - /// MultipleCreateOrderHelper, which indexes that dictionary by each relationship field. - /// Foreign keys inferred from the catalog are deliberately not checked: there is no - /// configuration to correct there, so refusing to start would punish the database's shape - /// rather than the configuration, and the relationship simply not forming is the - /// proportionate outcome. - /// - private void RejectConfiguredRelationshipReferencesToSkippedColumns() + private void ValidateReferencesToUnreadableColumns(string entityName, Entity entity) { - foreach ((string entityName, Entity entity) in Entities) + UnreadableColumns? unreadableColumns = GetUnreadableColumnsForEntity(entityName); + + if (unreadableColumns is not null) { - if (entity.Relationships is null) - { - continue; - } + ThrowIfReferencesUnreadableColumn(entityName, unreadableColumns, entity.Mappings?.Keys, "mappings"); + ThrowIfReferencesUnreadableColumn(entityName, unreadableColumns, entity.Fields?.Select(field => field.Name), "fields"); - foreach ((string relationshipName, EntityRelationship relationship) in entity.Relationships) + foreach (EntityPermission permission in entity.Permissions) { - RejectRelationshipFields( - entityName, - relationship.SourceFields, - $"source.fields of relationship {relationshipName} of entity {entityName}"); - - RejectRelationshipFields( - relationship.TargetEntity, - relationship.TargetFields, - $"target.fields of relationship {relationshipName} of entity {entityName}"); + foreach (EntityAction action in permission.Actions) + { + ThrowIfReferencesUnreadableColumn( + entityName, + unreadableColumns, + action.Fields?.Include, + $"fields.include of role '{permission.Role}'"); + + // Policies name exposed fields, like the OData model they are parsed against. + IEnumerable policyColumns = AuthorizationResolver + .GetFieldReferencesInDatabasePolicy(action.Policy?.Database) + .Select(field => TryGetBackingColumn(entityName, field, out string? backingColumn) ? backingColumn : field); + ThrowIfReferencesUnreadableColumn( + entityName, + unreadableColumns, + policyColumns, + $"the database policy of role '{permission.Role}'"); + } } } - } - /// - /// Fails initialization when any of the given fields, read as names configured on the given - /// entity, resolves to a column that entity's object does not expose. - /// - private void RejectRelationshipFields(string entityName, string[]? fields, string configurationSection) - { - if (fields is null - || fields.Length == 0 - || !Entities.TryGetValue(entityName, out Entity? entity) - || !EntityToDatabaseObject.TryGetValue(entityName, out DatabaseObject? databaseObject) - || !_skippedColumnsByObject.TryGetValue( - GetObjectCacheKey(databaseObject.SchemaName, databaseObject.Name), - out Dictionary? skippedColumns)) + if (entity.Relationships is null) { return; } - foreach (string field in fields) + // Relationship fields name backing columns, so they are compared as configured. + foreach ((string relationshipName, EntityRelationship relationship) in entity.Relationships) { - string backingColumn = ResolveBackingColumnName(entity, field); + ThrowIfReferencesUnreadableColumn( + entityName, + unreadableColumns, + relationship.SourceFields, + $"source.fields of relationship '{relationshipName}'"); + ThrowIfReferencesUnreadableColumn( + entityName, + GetUnreadableColumnsForEntity(relationship.TargetEntity), + relationship.TargetFields, + $"target.fields of relationship '{relationshipName}'"); - if (!skippedColumns.TryGetValue(backingColumn, out string? dataType)) + if (!string.IsNullOrWhiteSpace(relationship.LinkingObject)) { - continue; + (string linkingSchemaName, string linkingTableName) = ParseSchemaAndDbTableName(relationship.LinkingObject); + UnreadableColumns? linkingColumns = GetUnreadableColumns(linkingSchemaName, linkingTableName); + ThrowIfReferencesUnreadableColumn( + entityName, + linkingColumns, + relationship.LinkingSourceFields, + $"linking.source.fields of relationship '{relationshipName}'"); + ThrowIfReferencesUnreadableColumn( + entityName, + linkingColumns, + relationship.LinkingTargetFields, + $"linking.target.fields of relationship '{relationshipName}'"); } - - throw new DataApiBuilderException( - message: $"The {configurationSection} reference the column {backingColumn} of " - + $"{databaseObject.SchemaName}.{databaseObject.Name}, whose data type {dataType} is not " - + "supported. That column is not part of the exposed contract, so the relationship cannot be " - + "honored. Remove it from the configuration.", - statusCode: HttpStatusCode.ServiceUnavailable, - subStatusCode: DataApiBuilderException.SubStatusCodes.ErrorInInitialization); } } - /// - /// Resolves an exposed field name to the column it is backed by, using the aliases the - /// configuration declares through "mappings" and through a field's "alias". An unaliased - /// name is already the backing name. - /// - private static string ResolveBackingColumnName(Entity entity, string exposedName) + private static void ThrowIfReferencesUnreadableColumn( + string entityName, + UnreadableColumns? unreadableColumns, + IEnumerable? columnNames, + string configurationSection) { - // "fields" is consulted before "mappings", mirroring the precedence - // GenerateExposedToBackingColumnMapUtil applies when it builds the map the runtime - // resolves names through. Reversing it here would reject a configuration the runtime - // resolves the other way. - if (entity.Fields is not null) + if (unreadableColumns is null || columnNames is null) { - foreach (FieldMetadata field in entity.Fields) - { - if (!string.IsNullOrWhiteSpace(field.Alias) - && string.Equals(field.Alias, exposedName, StringComparison.OrdinalIgnoreCase)) - { - return field.Name; - } - } + return; } - if (entity.Mappings is not null) + foreach (string columnName in columnNames) { - foreach (KeyValuePair mapping in entity.Mappings) + if (unreadableColumns.TypeNameByColumn.TryGetValue(columnName, out string? typeName)) { - if (string.Equals(mapping.Value, exposedName, StringComparison.OrdinalIgnoreCase)) - { - return mapping.Key; - } + throw new DataApiBuilderException( + message: $"Entity '{entityName}': {configurationSection} references column '{columnName}', whose data type " + + $"'{typeName}' is not supported. Remove the reference or use a supported column.", + statusCode: HttpStatusCode.ServiceUnavailable, + subStatusCode: DataApiBuilderException.SubStatusCodes.ErrorInInitialization); } } - - return exposedName; - } - - /// - /// Renders skipped columns as "name (type)" pairs for log and error messages. - /// - private static string FormatSkippedColumns(Dictionary skippedColumns) - { - return string.Join(", ", skippedColumns.Select(entry => $"{entry.Key} ({entry.Value})")); - } - - /// - /// Key used by the per-object metadata caches held during initialization. The schema name is - /// length-prefixed rather than joined with a dot, because bracketed identifiers may contain - /// dots: [a.b].[c] and [a].[b.c] are different objects that a "schema.table" - /// key would collide, letting one reuse the other's catalog rows. - /// - private static string GetObjectCacheKey(string schemaName, string tableName) - { - return $"{schemaName.Length}:{schemaName}{tableName}"; - } - - /// - /// Releases the catalog metadata gathered during initialization. Nothing reads these caches - /// once object definitions are populated, and the cached tables are disposable. - /// - private void ReleaseCatalogMetadataCaches() - { - foreach (DataTable columnsInTable in _columnsMetadataCache.Values) - { - columnsInTable.Dispose(); - } - - _columnsMetadataCache.Clear(); - _skippedColumnsByObject.Clear(); - _objectCatalogMetadataCache.Clear(); - _unreachableKeyReasonByObject.Clear(); } /// - /// Returns the "Columns" schema collection for a database object, reading it from the - /// catalog once per object. Schema discovery and column definition population both need it, - /// and each call opens its own connection. + /// Columns a schema read left out because the data provider reports no CLR type for them. /// - private async Task GetCachedColumnsAsync( - string schemaName, - string tableName, - CancellationToken cancellationToken) - { - string cacheKey = GetObjectCacheKey(schemaName, tableName); - - if (_columnsMetadataCache.TryGetValue(cacheKey, out DataTable? cachedColumns)) - { - return cachedColumns; - } - - DataTable columnsInTable = await GetColumnsAsync(schemaName, tableName, cancellationToken); - _columnsMetadataCache[cacheKey] = columnsInTable; - - return columnsInTable; - } + /// Each skipped column with its database type name. + /// Skipped columns that belong to the key the data adapter would have inferred. + private sealed record UnreadableColumns( + IReadOnlyDictionary TypeNameByColumn, + IReadOnlyList KeyColumns); /// /// Gets the correctly formatted table name with schema as prefix, if one exists. From 05fcc1f97d4d4bf9e697c6e78fbf5f5878e8a30f Mon Sep 17 00:00:00 2001 From: Aaron Burtle Date: Wed, 11 Nov 2026 21:15:27 -0800 Subject: [PATCH 18/19] Rework tests for columns with unsupported data types Move the schema discovery tests into SqlMetadataProviderUnsupportedColumnTypeTests, assert the exact errors, expect the same keys as the data adapter infers, and cover relationship fields named like an alias, entities that name an object in different casing, and Type System Version=SQL Server 2005. Add REST and GraphQL tests for the GeometryType entity and unit tests for the policy field parser, and update the fixture comments. --- .../AuthorizationResolverUnitTests.cs | 10 + src/Service.Tests/DatabaseSchema-MsSql.sql | 39 +- .../MsSqlGraphQLQueryTests.cs | 32 + .../RestApiTests/Find/MsSqlFindApiTests.cs | 38 + .../RestApiTests/Put/MsSqlPutApiTests.cs | 9 +- .../UnitTests/SqlMetadataProviderUnitTests.cs | 1075 ----------------- ...adataProviderUnsupportedColumnTypeTests.cs | 298 +++++ 7 files changed, 391 insertions(+), 1110 deletions(-) create mode 100644 src/Service.Tests/UnitTests/SqlMetadataProviderUnsupportedColumnTypeTests.cs diff --git a/src/Service.Tests/Authorization/AuthorizationResolverUnitTests.cs b/src/Service.Tests/Authorization/AuthorizationResolverUnitTests.cs index b01d095fa1..f541155234 100644 --- a/src/Service.Tests/Authorization/AuthorizationResolverUnitTests.cs +++ b/src/Service.Tests/Authorization/AuthorizationResolverUnitTests.cs @@ -1982,6 +1982,16 @@ public void ValidateUnauthenticatedUserClaimsAreNotResolvedWhenProcessingUserCla bool didResolveUnauthenticatedOidClaim = resolvedClaims["oid"] == "1337"; Assert.AreEqual(expected: false, actual: didResolveUnauthenticatedRoleClaim, message: "Unauthenticated oid claim erroneously resolved."); } + + [DataTestMethod] + [DataRow(null, "", DisplayName = "No policy")] + [DataRow("@item.id eq @claims.userId and @item.owner_id ne 1", "id,owner_id", DisplayName = "Fields next to a claim")] + [DataRow("@item.name eq '@item.geom'", "name", DisplayName = "Reference inside a string literal")] + [DataRow("@item.name ne 'it''s @item.geom' or @item.id gt 1", "name,id", DisplayName = "Escaped quote inside a string literal")] + public void GetFieldReferencesInDatabasePolicySkipsStringLiterals(string policy, string expectedFields) + { + Assert.AreEqual(expectedFields, string.Join(",", AuthorizationResolver.GetFieldReferencesInDatabasePolicy(policy))); + } #endregion #region Helpers diff --git a/src/Service.Tests/DatabaseSchema-MsSql.sql b/src/Service.Tests/DatabaseSchema-MsSql.sql index f91c1ba0cd..7562e6e6aa 100644 --- a/src/Service.Tests/DatabaseSchema-MsSql.sql +++ b/src/Service.Tests/DatabaseSchema-MsSql.sql @@ -277,8 +277,7 @@ CREATE TABLE geometry_type_table( geom geometry NULL ); --- The period columns are HIDDEN, so SELECT * does not return them. An explicit projection built --- from the catalog would, which is what this fixture guards against. +-- The period columns are HIDDEN, so SELECT * doesn't return them and they must stay out of the exposed columns. CREATE TABLE temporal_geometry_type_table( id int NOT NULL PRIMARY KEY, name varchar(100) NOT NULL, @@ -289,8 +288,7 @@ CREATE TABLE temporal_geometry_type_table( ) WITH (SYSTEM_VERSIONING = ON (HISTORY_TABLE = dbo.temporal_geometry_type_table_history)); --- The database primary key is itself of an unsupported type, so no projection can expose the --- object: the engine needs the key it cannot read. +-- The primary key has an unsupported type, so the object needs source.key-fields to be exposed. CREATE TABLE hierarchyid_pk_table( node hierarchyid NOT NULL PRIMARY KEY, name varchar(100) NOT NULL @@ -304,9 +302,7 @@ CREATE TABLE hierarchyid_composite_pk_table( CONSTRAINT PK_hierarchyid_composite_pk_table PRIMARY KEY (tenant_id, node) ); --- No database primary key, and the unsupported column carries a unique index. This object is --- expected to load, with the key supplied through source.key-fields: the data adapter's own key --- discovery would otherwise pull the unique column back in as a hidden reader column. +-- No primary key, and the only unique key has an unsupported type, so the object needs source.key-fields. CREATE TABLE hierarchyid_unique_table( id int NOT NULL, node hierarchyid NOT NULL, @@ -314,9 +310,7 @@ CREATE TABLE hierarchyid_unique_table( CONSTRAINT UQ_hierarchyid_unique_table_node UNIQUE (node) ); --- No database primary key, and the unique index covers a supported non-null column. The data --- adapter promotes such a key to DataTable.PrimaryKey on the unnarrowed path, so the narrowed path --- has to infer it too instead of demanding source.key-fields. +-- No primary key, but a unique key over a supported non-null column, which is inferred as the key. CREATE TABLE unique_key_geometry_table( code varchar(20) NOT NULL, name varchar(100) NOT NULL, @@ -324,9 +318,7 @@ CREATE TABLE unique_key_geometry_table( CONSTRAINT UQ_unique_key_geometry_table_code UNIQUE (code) ); --- A primary key and a separate non-null unique key. A view over this table that does not select the --- primary key is still keyable through the unique one, which is ordinary design rather than a key --- the object cannot express. +-- A primary key and a separate unique key, for a view that selects only the unique one. CREATE TABLE pk_and_unique_geometry_table( id int IDENTITY(1, 1) NOT NULL PRIMARY KEY, code varchar(20) NOT NULL, @@ -335,8 +327,7 @@ CREATE TABLE pk_and_unique_geometry_table( CONSTRAINT UQ_pk_and_unique_geometry_table_code UNIQUE (code) ); --- A decimal identity column. DataColumn.AutoIncrement would coerce its CLR type to Int32, which is --- why identity is carried from the catalog instead; this fixture is what proves the type survives. +-- A decimal identity column, whose type and identity must survive skipping the geometry column. CREATE TABLE decimal_identity_geometry_table( id decimal(18, 0) IDENTITY(1, 1) NOT NULL PRIMARY KEY, name varchar(100) NOT NULL, @@ -882,23 +873,15 @@ EXEC('CREATE VIEW books_publishers_view_composite_insertable as SELECT FROM dbo.books,dbo.publishers where publishers.id = books.publisher_id'); EXEC('CREATE VIEW geometry_only_view AS SELECT geom FROM dbo.geometry_type_table'); --- A view has no index of its own, so its key can only be resolved through the underlying table. +-- A view's key is inferred from the underlying table. EXEC('CREATE VIEW geometry_all_view AS SELECT id, name, geom FROM dbo.geometry_type_table'); --- The underlying key is composite and one of its members is of an unsupported type, so no key the --- projection can express identifies a row. Reporting the readable member alone would be a partial --- key, which silently matches more than one row on an update or a delete. +-- One member of the underlying composite key has an unsupported type, and the other alone isn't a key. EXEC('CREATE VIEW hierarchyid_composite_view AS SELECT tenant_id, name, node FROM dbo.hierarchyid_composite_pk_table'); --- Omits the primary key of the underlying table and carries its unique key whole. +-- Omits the underlying primary key, so no key is inferred even though the unique key is selected. EXEC('CREATE VIEW unique_without_pk_view AS SELECT code, name, geom FROM dbo.pk_and_unique_geometry_table'); --- A join whose second object contributes only the unsupported column. Once that column is dropped --- from the projection, every column the projection selects resolves to the first object, and only --- the hidden key column browse mode reports for the second one reveals that this is a join at all. --- A key inferred from the first object alone would not identify a row, since the join can duplicate --- its rows. +-- A join whose second table contributes only the unsupported column. EXEC('CREATE VIEW join_geometry_view AS SELECT a.id, a.name, b.geom FROM dbo.geometry_type_table a JOIN dbo.unique_key_geometry_table b ON b.name = a.name'); --- A self-join, where browse mode reports the same physical table for both instances and so cannot --- be told apart from a single one by source lineage alone. What does give it away is the primary --- key column being reported twice, once for each instance. +-- A self-join, which selects the key of only one of the two instances. EXEC('CREATE VIEW self_join_geometry_view AS SELECT a.id, a.name, b.geom FROM dbo.geometry_type_table a JOIN dbo.geometry_type_table b ON b.name = a.name'); EXEC('CREATE PROCEDURE get_book_by_id @id int AS SELECT * FROM dbo.books diff --git a/src/Service.Tests/SqlTests/GraphQLQueryTests/MsSqlGraphQLQueryTests.cs b/src/Service.Tests/SqlTests/GraphQLQueryTests/MsSqlGraphQLQueryTests.cs index a9578137b7..87da704545 100644 --- a/src/Service.Tests/SqlTests/GraphQLQueryTests/MsSqlGraphQLQueryTests.cs +++ b/src/Service.Tests/SqlTests/GraphQLQueryTests/MsSqlGraphQLQueryTests.cs @@ -807,6 +807,38 @@ public async Task TestStoredProcedureQueryWithNoDefaultInConfig() SqlTestHelper.TestForErrorInGraphQLResponse(result.ToString(), message: "The argument `id` is required."); } + /// + /// A table with a geometry column, which the data provider can't read, is queryable without that column. + /// + [TestMethod] + public async Task QueryTableWithUnsupportedColumnType() + { + string graphQLQuery = @"{ + geometryTypes(orderBy: { id: ASC }) { + items { + id + name + } + } + }"; + + JsonElement actual = await ExecuteGraphQLRequestAsync(graphQLQuery, "geometryTypes", isAuthenticated: false); + string expected = await GetDatabaseResultAsync( + "SELECT [id], [name] FROM geometry_type_table ORDER BY [id] FOR JSON PATH, INCLUDE_NULL_VALUES"); + SqlTestHelper.PerformTestEqualJsonStrings(expected, actual.GetProperty("items").ToString()); + + graphQLQuery = @"{ + geometryType_by_pk(id: 1) { + geom + } + }"; + + JsonElement error = await ExecuteGraphQLRequestAsync(graphQLQuery, "geometryType_by_pk", isAuthenticated: false); + SqlTestHelper.TestForErrorInGraphQLResponse( + error.ToString(), + message: "The field `geom` does not exist on the type `GeometryType`."); + } + /// /// Test to check GraphQL support for aggregations with aliases. /// This test verifies that the SQL query results are correctly mapped to the expected GraphQL format. diff --git a/src/Service.Tests/SqlTests/RestApiTests/Find/MsSqlFindApiTests.cs b/src/Service.Tests/SqlTests/RestApiTests/Find/MsSqlFindApiTests.cs index 651bfe812a..c284930757 100644 --- a/src/Service.Tests/SqlTests/RestApiTests/Find/MsSqlFindApiTests.cs +++ b/src/Service.Tests/SqlTests/RestApiTests/Find/MsSqlFindApiTests.cs @@ -591,6 +591,16 @@ public class MsSqlFindApiTests : FindApiTestBase $"SELECT * FROM { _integrationBrokenMappingTable } " + $"WHERE habitat = 'forestland' " + $"FOR JSON PATH, INCLUDE_NULL_VALUES" + }, + { + "FindOnTableWithUnsupportedColumnType", + $"SELECT [id], [name] FROM geometry_type_table " + + $"ORDER BY [id] FOR JSON PATH, INCLUDE_NULL_VALUES" + }, + { + "FindByIdOnTableWithUnsupportedColumnType", + $"SELECT [id], [name] FROM geometry_type_table " + + $"WHERE [id] = 1 FOR JSON PATH, INCLUDE_NULL_VALUES, WITHOUT_ARRAY_WRAPPER" } }; #region Test Fixture Setup @@ -672,6 +682,34 @@ await SetupAndRunRestApiTest( #region RestApiTests Outliers + /// + /// A table with a geometry column, which the data provider can't read, is served without that column. + /// + [TestMethod] + public async Task FindOnTableWithUnsupportedColumnType() + { + await SetupAndRunRestApiTest( + primaryKeyRoute: string.Empty, + queryString: string.Empty, + entityNameOrPath: "GeometryType", + sqlQuery: GetQuery(nameof(FindOnTableWithUnsupportedColumnType))); + + await SetupAndRunRestApiTest( + primaryKeyRoute: "id/1", + queryString: string.Empty, + entityNameOrPath: "GeometryType", + sqlQuery: GetQuery("FindByIdOnTableWithUnsupportedColumnType")); + + await SetupAndRunRestApiTest( + primaryKeyRoute: "id/1", + queryString: "?$select=geom", + entityNameOrPath: "GeometryType", + sqlQuery: string.Empty, + exceptionExpected: true, + expectedErrorMessage: "Invalid field to be returned requested: geom", + expectedStatusCode: HttpStatusCode.BadRequest); + } + /// /// Tests we ensure that an invalid query in the EasyAuth header /// retruns a successful request without executing the invalid query. diff --git a/src/Service.Tests/SqlTests/RestApiTests/Put/MsSqlPutApiTests.cs b/src/Service.Tests/SqlTests/RestApiTests/Put/MsSqlPutApiTests.cs index 63d9bae9ec..8f27d9ef0e 100644 --- a/src/Service.Tests/SqlTests/RestApiTests/Put/MsSqlPutApiTests.cs +++ b/src/Service.Tests/SqlTests/RestApiTests/Put/MsSqlPutApiTests.cs @@ -334,13 +334,8 @@ public async Task PutOneInViewBadRequest() } /// - /// Test to validate successful execution of a PUT against a system-versioned temporal table - /// whose period columns are declared GENERATED ALWAYS ... HIDDEN. - /// PUT has overwrite semantics and nulls the writable columns a request body leaves out, so a - /// period column that reached the exposed contract would be targeted and the request would - /// fail with "Cannot update GENERATED ALWAYS columns". Successful execution confirms those - /// columns are not part of the contract: "SELECT *" does not return them, and schema - /// discovery does not name them either. + /// PUT nulls the writable columns a request leaves out, so on this temporal table, which also has a skipped + /// geometry column, it succeeds only if the HIDDEN period columns stay out of the entity's columns. /// [TestMethod] public async Task PutOneWithHiddenPeriodColumnsInTemporalTable() diff --git a/src/Service.Tests/UnitTests/SqlMetadataProviderUnitTests.cs b/src/Service.Tests/UnitTests/SqlMetadataProviderUnitTests.cs index 294a1a69a5..8dae4104b7 100644 --- a/src/Service.Tests/UnitTests/SqlMetadataProviderUnitTests.cs +++ b/src/Service.Tests/UnitTests/SqlMetadataProviderUnitTests.cs @@ -397,1081 +397,6 @@ public async Task ValidateInferredRelationshipInfoForMsSql() ValidateInferredRelationshipInfoForTables(); } - /// - /// Test to validate that a table holding a column whose data type the data provider cannot - /// map to a CLR type - here a geometry column - is still usable: metadata inference must - /// succeed and the unsupported column must be absent from the inferred source definition, - /// so it never reaches the OData or GraphQL type maps. - /// The entity places no field restriction, so this covers the column being skipped on the - /// strength of its type alone. - /// - [TestMethod, TestCategory(TestCategory.MSSQL)] - public async Task ValidateUnsupportedColumnTypeIsNotInferred() - { - DatabaseEngine = TestCategory.MSSQL; - await SetupTestFixtureAndInferMetadata(); - - Assert.IsTrue( - _sqlMetadataProvider.GetEntityNamesAndDbObjects().TryGetValue("GeometryType", out DatabaseObject databaseObject), - message: "Metadata inference failed for the entity backed by a table with a geometry column."); - - SourceDefinition sourceDefinition = databaseObject.SourceDefinition; - - Assert.IsTrue( - sourceDefinition.Columns.ContainsKey("id"), - message: "The primary key column is expected in the source definition."); - Assert.IsTrue( - sourceDefinition.Columns.ContainsKey("name"), - message: "A column with a supported data type is expected in the source definition."); - Assert.IsFalse( - sourceDefinition.Columns.ContainsKey("geom"), - message: "A column whose data type cannot be mapped is not expected in the source definition."); - - // Identity is carried from the catalog on the narrowed path rather than through - // DataColumn.AutoIncrement. Asserted here because losing it would silently make creates - // require an identity value. - Assert.IsTrue( - sourceDefinition.Columns["id"].IsAutoGenerated, - message: "The identity column is expected to be marked auto-generated."); - Assert.IsTrue( - sourceDefinition.Columns["id"].IsReadOnly, - message: "An auto-generated column is expected to be read-only."); - - TestHelper.UnsetAllDABEnvironmentVariables(); - } - - /// - /// Test to validate that an identity column whose CLR type cannot be auto-incremented keeps - /// the type the provider reported. - /// `DataColumn.AutoIncrement` coerces such a DataType to Int32, and SQL Server allows - /// identity on tinyint, numeric and decimal, so carrying the flag through that property - /// would report the wrong SystemType and reach parameter typing and the generated API - /// schemas. This is the fixture that proves the type survives. - /// - [TestMethod, TestCategory(TestCategory.MSSQL)] - public async Task ValidateIdentityTypeIsPreservedOnTheNarrowedPath() - { - DatabaseEngine = TestCategory.MSSQL; - TestHelper.SetupDatabaseEnvironment(DatabaseEngine); - - await SetUpSingleEntityMetadataProviderAsync( - "DecimalIdentityGeometry", - BuildReadOnlyEntity( - entityName: "DecimalIdentityGeometry", - databaseObject: "dbo.decimal_identity_geometry_table", - sourceType: EntitySourceType.Table, - keyFields: null)); - - await _sqlMetadataProvider.InitializeAsync(); - - Assert.IsTrue( - _sqlMetadataProvider.GetEntityNamesAndDbObjects().TryGetValue("DecimalIdentityGeometry", out DatabaseObject databaseObject), - message: "Metadata inference failed for an object with a decimal identity column."); - - SourceDefinition sourceDefinition = databaseObject.SourceDefinition; - - Assert.AreEqual( - typeof(decimal), - sourceDefinition.Columns["id"].SystemType, - message: "The identity column is expected to keep the data type the provider reported."); - Assert.IsTrue( - sourceDefinition.Columns["id"].IsAutoGenerated, - message: "The identity column is expected to be marked auto-generated."); - Assert.IsTrue( - sourceDefinition.Columns["id"].IsReadOnly, - message: "An auto-generated column is expected to be read-only."); - Assert.IsFalse( - sourceDefinition.Columns.ContainsKey("geom"), - message: "A column whose data type cannot be mapped is not expected in the source definition."); - - TestHelper.UnsetAllDABEnvironmentVariables(); - } - - /// - /// Test to validate that a database policy referencing a column left out of the projection - /// fails initialization. - /// A policy is parsed per request against the OData model, which is built from - /// `SourceDefinition.Columns`, so a policy naming an absent column returns 400 on every - /// request for that role instead of the configuration being rejected once, at startup. - /// - [TestMethod, TestCategory(TestCategory.MSSQL)] - public async Task ValidateDatabasePolicyOverUnsupportedColumnFailsInitialization() - { - DatabaseEngine = TestCategory.MSSQL; - TestHelper.SetupDatabaseEnvironment(DatabaseEngine); - - Entity entity = new( - Source: new("dbo.geometry_type_table", EntitySourceType.Table, null, new string[] { "id" }), - Fields: null, - Rest: new(Enabled: true), - GraphQL: new("GeometryPolicy", "GeometryPolicys", Enabled: true), - Permissions: new EntityPermission[] - { - new(Role: "anonymous", - Actions: new EntityAction[] - { - new(Action: EntityActionOperation.Read, - Fields: null, - Policy: new(Request: null, Database: "@item.geom eq null")) - }) - }, - Relationships: null, - Mappings: null); - - await SetUpSingleEntityMetadataProviderAsync("GeometryPolicy", entity); - - try - { - await _sqlMetadataProvider.InitializeAsync(); - Assert.Fail("Expected DataApiBuilderException was not thrown for a database policy over a column of an unsupported data type."); - } - catch (DataApiBuilderException ex) - { - Assert.AreEqual(HttpStatusCode.ServiceUnavailable, ex.StatusCode); - Assert.AreEqual(DataApiBuilderException.SubStatusCodes.ErrorInInitialization, ex.SubStatusCode); - Assert.IsTrue( - ex.Message.Contains("geom") && ex.Message.Contains("policy"), - message: $"The error is expected to name the column and the policy referencing it. Actual message: {ex.Message}"); - } - - TestHelper.UnsetAllDABEnvironmentVariables(); - } - - /// - /// Test to validate that a database object whose every column has a data type the data - /// provider cannot map fails initialization with a specific error, rather than falling back - /// to reading every column and surfacing the provider's opaque failure instead of the reason. - /// The entity is declared in an in-memory config rather than in dab-config.MsSql.json, - /// because this object fails by design and every MSSQL fixture initializes every configured - /// entity. - /// - [TestMethod, TestCategory(TestCategory.MSSQL)] - public async Task ValidateObjectWithOnlyUnsupportedColumnsFailsInitialization() - { - DatabaseEngine = TestCategory.MSSQL; - TestHelper.SetupDatabaseEnvironment(DatabaseEngine); - - Dictionary entities = new() - { - { - "GeometryOnlyView", - new Entity( - Source: new("dbo.geometry_only_view", EntitySourceType.View, null, new string[] { "geom" }), - Fields: null, - Rest: new(Enabled: true), - GraphQL: new("GeometryOnlyView", "GeometryOnlyViews", Enabled: true), - Permissions: new EntityPermission[] - { - new(Role: "anonymous", - Actions: new EntityAction[] { new(Action: EntityActionOperation.Read, Fields: null, Policy: null) }) - }, - Relationships: null, - Mappings: null) - } - }; - - RuntimeConfig runtimeConfig = SqlTestHelper.SetupRuntimeConfig() with { Entities = new RuntimeEntities(entities) }; - RuntimeConfigProvider runtimeConfigProvider = TestHelper.GenerateInMemoryRuntimeConfigProvider(runtimeConfig); - SetUpSQLMetadataProvider(runtimeConfigProvider); - await ResetDbStateAsync(); - - try - { - await _sqlMetadataProvider.InitializeAsync(); - Assert.Fail("Expected DataApiBuilderException was not thrown for an object whose every column has an unsupported data type."); - } - catch (DataApiBuilderException ex) - { - Assert.AreEqual(HttpStatusCode.ServiceUnavailable, ex.StatusCode); - Assert.AreEqual(DataApiBuilderException.SubStatusCodes.ErrorInInitialization, ex.SubStatusCode); - Assert.IsTrue( - ex.Message.Contains("has a data type that is not supported"), - message: $"Unexpected exception message: {ex.Message}"); - } - - TestHelper.UnsetAllDABEnvironmentVariables(); - } - - /// - /// Test to validate that a configured primary key naming a column of an unsupported data - /// type fails initialization instead of producing a source definition whose primary key is - /// absent from its columns. The error names both the column and the offending type. - /// The entity is declared in an in-memory config for the same reason as the test above. - /// - [TestMethod, TestCategory(TestCategory.MSSQL)] - public async Task ValidatePrimaryKeyOnUnsupportedColumnFailsInitialization() - { - DatabaseEngine = TestCategory.MSSQL; - TestHelper.SetupDatabaseEnvironment(DatabaseEngine); - - Dictionary entities = new() - { - { - "GeometryKeyed", - new Entity( - Source: new("dbo.geometry_type_table", EntitySourceType.Table, null, new string[] { "geom" }), - Fields: null, - Rest: new(Enabled: true), - GraphQL: new("GeometryKeyed", "GeometryKeyeds", Enabled: true), - Permissions: new EntityPermission[] - { - new(Role: "anonymous", - Actions: new EntityAction[] { new(Action: EntityActionOperation.Read, Fields: null, Policy: null) }) - }, - Relationships: null, - Mappings: null) - } - }; - - RuntimeConfig runtimeConfig = SqlTestHelper.SetupRuntimeConfig() with { Entities = new RuntimeEntities(entities) }; - RuntimeConfigProvider runtimeConfigProvider = TestHelper.GenerateInMemoryRuntimeConfigProvider(runtimeConfig); - SetUpSQLMetadataProvider(runtimeConfigProvider); - await ResetDbStateAsync(); - - try - { - await _sqlMetadataProvider.InitializeAsync(); - Assert.Fail("Expected DataApiBuilderException was not thrown for a primary key of an unsupported data type."); - } - catch (DataApiBuilderException ex) - { - Assert.AreEqual(HttpStatusCode.ServiceUnavailable, ex.StatusCode); - Assert.AreEqual(DataApiBuilderException.SubStatusCodes.ErrorInInitialization, ex.SubStatusCode); - Assert.IsTrue( - ex.Message.Contains("geom") && ex.Message.Contains("geometry"), - message: $"The error is expected to name the column and its data type. Actual message: {ex.Message}"); - } - - TestHelper.UnsetAllDABEnvironmentVariables(); - } - - /// - /// Test to validate that the period columns of a temporal table declared - /// GENERATED ALWAYS ... HIDDEN stay out of the inferred source definition. - /// "SELECT *" does not return them, so an explicit projection built from the catalog must not - /// name them either: adding them widens the exposed contract, and the read-only - /// classification does not recognize generated-always period columns, so an overwriting PUT - /// would try to null them and fail. - /// - [TestMethod, TestCategory(TestCategory.MSSQL)] - public async Task ValidateHiddenPeriodColumnsAreNotInferred() - { - DatabaseEngine = TestCategory.MSSQL; - TestHelper.SetupDatabaseEnvironment(DatabaseEngine); - - await SetUpSingleEntityMetadataProviderAsync( - "TemporalGeometryType", - BuildReadOnlyEntity( - entityName: "TemporalGeometryType", - databaseObject: "dbo.temporal_geometry_type_table", - sourceType: EntitySourceType.Table, - keyFields: new string[] { "id" })); - - await _sqlMetadataProvider.InitializeAsync(); - - Assert.IsTrue( - _sqlMetadataProvider.GetEntityNamesAndDbObjects().TryGetValue("TemporalGeometryType", out DatabaseObject databaseObject), - message: "Metadata inference failed for the entity backed by a temporal table."); - - SourceDefinition sourceDefinition = databaseObject.SourceDefinition; - - Assert.IsTrue( - sourceDefinition.Columns.ContainsKey("id"), - message: "The configured key column is expected in the source definition."); - Assert.IsTrue( - sourceDefinition.Columns.ContainsKey("name"), - message: "A column with a supported data type is expected in the source definition."); - Assert.IsFalse( - sourceDefinition.Columns.ContainsKey("geom"), - message: "A column whose data type cannot be mapped is not expected in the source definition."); - Assert.IsFalse( - sourceDefinition.Columns.ContainsKey("valid_from"), - message: "A HIDDEN period column is not returned by SELECT * and is not expected in the source definition."); - Assert.IsFalse( - sourceDefinition.Columns.ContainsKey("valid_to"), - message: "A HIDDEN period column is not returned by SELECT * and is not expected in the source definition."); - - TestHelper.UnsetAllDABEnvironmentVariables(); - } - - /// - /// Test to validate that an object whose own database primary key is a column of an - /// unsupported data type fails initialization with the reason. - /// The projection cannot carry that column, and the engine cannot operate on the object - /// without its key, so the object is unreachable either way. What this asserts is that the - /// failure names the column and its type instead of reporting a missing primary key, which - /// reads as something the user forgot to configure. - /// - [TestMethod, TestCategory(TestCategory.MSSQL)] - public async Task ValidateUnsupportedDatabasePrimaryKeyFailsInitialization() - { - DatabaseEngine = TestCategory.MSSQL; - TestHelper.SetupDatabaseEnvironment(DatabaseEngine); - - await SetUpSingleEntityMetadataProviderAsync( - "HierarchyIdKeyed", - BuildReadOnlyEntity( - entityName: "HierarchyIdKeyed", - databaseObject: "dbo.hierarchyid_pk_table", - sourceType: EntitySourceType.Table, - keyFields: null)); - - try - { - await _sqlMetadataProvider.InitializeAsync(); - Assert.Fail("Expected DataApiBuilderException was not thrown for a database primary key of an unsupported data type."); - } - catch (DataApiBuilderException ex) - { - Assert.AreEqual(HttpStatusCode.ServiceUnavailable, ex.StatusCode); - Assert.AreEqual(DataApiBuilderException.SubStatusCodes.ErrorInInitialization, ex.SubStatusCode); - Assert.IsTrue( - ex.Message.Contains("node") && ex.Message.Contains("hierarchyid"), - message: $"The error is expected to name the key column and its data type. Actual message: {ex.Message}"); - } - - TestHelper.UnsetAllDABEnvironmentVariables(); - } - - /// - /// Test to validate the same rejection when the unsupported column is one member of a - /// composite database primary key. The supported member alone does not identify a row, so - /// the object cannot be exposed through a partial key. - /// - [TestMethod, TestCategory(TestCategory.MSSQL)] - public async Task ValidateUnsupportedColumnInCompositeDatabasePrimaryKeyFailsInitialization() - { - DatabaseEngine = TestCategory.MSSQL; - TestHelper.SetupDatabaseEnvironment(DatabaseEngine); - - await SetUpSingleEntityMetadataProviderAsync( - "HierarchyIdCompositeKeyed", - BuildReadOnlyEntity( - entityName: "HierarchyIdCompositeKeyed", - databaseObject: "dbo.hierarchyid_composite_pk_table", - sourceType: EntitySourceType.Table, - keyFields: null)); - - try - { - await _sqlMetadataProvider.InitializeAsync(); - Assert.Fail("Expected DataApiBuilderException was not thrown for a composite database primary key holding an unsupported data type."); - } - catch (DataApiBuilderException ex) - { - Assert.AreEqual(HttpStatusCode.ServiceUnavailable, ex.StatusCode); - Assert.AreEqual(DataApiBuilderException.SubStatusCodes.ErrorInInitialization, ex.SubStatusCode); - Assert.IsTrue( - ex.Message.Contains("node") && ex.Message.Contains("hierarchyid"), - message: $"The error is expected to name the key column and its data type. Actual message: {ex.Message}"); - } - - TestHelper.UnsetAllDABEnvironmentVariables(); - } - - /// - /// Test to validate that an object carrying a unique index over a column of an unsupported - /// data type loads when a supported key is configured through source.key-fields. - /// This is the case the data adapter cannot serve: FillSchema runs with - /// CommandBehavior.KeyInfo, under which the provider performs its own key discovery and - /// appends key columns missing from the SELECT list as hidden reader columns - so the - /// unsupported unique column comes back regardless of the projection, and configuring a - /// supported key does not change that. Schema discovery therefore reads the shape without - /// KeyInfo once the projection is narrowed, and takes the primary key from the catalog. - /// - [TestMethod, TestCategory(TestCategory.MSSQL)] - public async Task ValidateConfiguredKeyIsUsedWhenUniqueIndexColumnIsUnsupported() - { - DatabaseEngine = TestCategory.MSSQL; - TestHelper.SetupDatabaseEnvironment(DatabaseEngine); - - await SetUpSingleEntityMetadataProviderAsync( - "HierarchyIdUnique", - BuildReadOnlyEntity( - entityName: "HierarchyIdUnique", - databaseObject: "dbo.hierarchyid_unique_table", - sourceType: EntitySourceType.Table, - keyFields: new string[] { "id" })); - - await _sqlMetadataProvider.InitializeAsync(); - - Assert.IsTrue( - _sqlMetadataProvider.GetEntityNamesAndDbObjects().TryGetValue("HierarchyIdUnique", out DatabaseObject databaseObject), - message: "Metadata inference failed for an object whose unique index covers an unsupported column."); - - SourceDefinition sourceDefinition = databaseObject.SourceDefinition; - - Assert.IsTrue( - sourceDefinition.Columns.ContainsKey("id"), - message: "The configured key column is expected in the source definition."); - Assert.IsTrue( - sourceDefinition.Columns.ContainsKey("name"), - message: "A column with a supported data type is expected in the source definition."); - Assert.IsFalse( - sourceDefinition.Columns.ContainsKey("node"), - message: "The unsupported unique column is not expected in the source definition."); - CollectionAssert.AreEqual( - new List { "id" }, - sourceDefinition.PrimaryKey, - message: "The configured key is expected to be the primary key in effect."); - - TestHelper.UnsetAllDABEnvironmentVariables(); - } - - /// - /// Test to validate that configuration naming a column left out of the projection fails - /// initialization. - /// Such a name keeps resolving after the column is gone, because the exposed and backing - /// column maps are built from entity fields and mappings without requiring the column to - /// exist in the source definition. The reference then reaches code that indexes the source - /// definition columns and fails per request rather than at startup. - /// - [TestMethod, TestCategory(TestCategory.MSSQL)] - public async Task ValidateConfiguredReferenceToUnsupportedColumnFailsInitialization() - { - DatabaseEngine = TestCategory.MSSQL; - TestHelper.SetupDatabaseEnvironment(DatabaseEngine); - - await SetUpSingleEntityMetadataProviderAsync( - "GeometryAliased", - BuildReadOnlyEntity( - entityName: "GeometryAliased", - databaseObject: "dbo.geometry_type_table", - sourceType: EntitySourceType.Table, - keyFields: new string[] { "id" }, - mappings: new Dictionary { { "geom", "Position" } })); - - try - { - await _sqlMetadataProvider.InitializeAsync(); - Assert.Fail("Expected DataApiBuilderException was not thrown for a mapping over a column of an unsupported data type."); - } - catch (DataApiBuilderException ex) - { - Assert.AreEqual(HttpStatusCode.ServiceUnavailable, ex.StatusCode); - Assert.AreEqual(DataApiBuilderException.SubStatusCodes.ErrorInInitialization, ex.SubStatusCode); - Assert.IsTrue( - ex.Message.Contains("geom") && ex.Message.Contains("mappings"), - message: $"The error is expected to name the column and the configuration section referencing it. Actual message: {ex.Message}"); - } - - TestHelper.UnsetAllDABEnvironmentVariables(); - } - - /// - /// Test to validate that an object with no database primary key, whose unique index covers a - /// supported non-null column, has that key inferred without `source.key-fields`. - /// `DbDataAdapter.FillSchema` promotes such a unique key to `DataTable.PrimaryKey` on the - /// unnarrowed path, so dropping it once the projection is narrowed would make the two paths - /// disagree and demand configuration for an object the other path resolves on its own. - /// - [TestMethod, TestCategory(TestCategory.MSSQL)] - public async Task ValidateUniqueKeyIsInferredWhenNoDatabasePrimaryKeyExists() - { - DatabaseEngine = TestCategory.MSSQL; - TestHelper.SetupDatabaseEnvironment(DatabaseEngine); - - await SetUpSingleEntityMetadataProviderAsync( - "UniqueKeyGeometry", - BuildReadOnlyEntity( - entityName: "UniqueKeyGeometry", - databaseObject: "dbo.unique_key_geometry_table", - sourceType: EntitySourceType.Table, - keyFields: null)); - - await _sqlMetadataProvider.InitializeAsync(); - - Assert.IsTrue( - _sqlMetadataProvider.GetEntityNamesAndDbObjects().TryGetValue("UniqueKeyGeometry", out DatabaseObject databaseObject), - message: "Metadata inference failed for an object whose only key is a unique index over a supported column."); - - SourceDefinition sourceDefinition = databaseObject.SourceDefinition; - - CollectionAssert.AreEqual( - new List { "code" }, - sourceDefinition.PrimaryKey, - message: "The non-null unique column is expected to be inferred as the primary key."); - Assert.IsTrue( - sourceDefinition.Columns.ContainsKey("name"), - message: "A column with a supported data type is expected in the source definition."); - Assert.IsFalse( - sourceDefinition.Columns.ContainsKey("geom"), - message: "A column whose data type cannot be mapped is not expected in the source definition."); - - TestHelper.UnsetAllDABEnvironmentVariables(); - } - - /// - /// Test to validate that a database policy whose text merely contains "@item." inside a - /// string literal is not read as a field reference. - /// The policy below compares a literal to itself, so the two occurrences of "@item.geom" are - /// data, not column references, and the entity has to start. - /// - [TestMethod, TestCategory(TestCategory.MSSQL)] - public async Task ValidateQuotedLiteralInDatabasePolicyIsNotAFieldReference() - { - DatabaseEngine = TestCategory.MSSQL; - TestHelper.SetupDatabaseEnvironment(DatabaseEngine); - - Entity entity = new( - Source: new("dbo.geometry_type_table", EntitySourceType.Table, null, new string[] { "id" }), - Fields: null, - Rest: new(Enabled: true), - GraphQL: new("GeometryQuotedPolicy", "GeometryQuotedPolicys", Enabled: true), - Permissions: new EntityPermission[] - { - new(Role: "anonymous", - Actions: new EntityAction[] - { - new(Action: EntityActionOperation.Read, - Fields: null, - Policy: new(Request: null, Database: "@item.id gt 0 and '@item.geom' eq '@item.geom'")) - }) - }, - Relationships: null, - Mappings: null); - - await SetUpSingleEntityMetadataProviderAsync("GeometryQuotedPolicy", entity); - - await _sqlMetadataProvider.InitializeAsync(); - - Assert.IsTrue( - _sqlMetadataProvider.GetEntityNamesAndDbObjects().ContainsKey("GeometryQuotedPolicy"), - message: "A policy containing the field prefix only inside a string literal must not be read as a reference."); - - TestHelper.UnsetAllDABEnvironmentVariables(); - } - - /// - /// Test to validate that a policy referencing an alias of a supported column is honored even - /// when the alias carries the name of a skipped column. - /// Policy identifiers are exposed names, so "geom" here is the alias of the supported "name" - /// column and not the skipped spatial column that happens to share the name. - /// - [TestMethod, TestCategory(TestCategory.MSSQL)] - public async Task ValidateDatabasePolicyOverAliasOfSupportedColumnIsHonored() - { - DatabaseEngine = TestCategory.MSSQL; - TestHelper.SetupDatabaseEnvironment(DatabaseEngine); - - Entity entity = new( - Source: new("dbo.geometry_type_table", EntitySourceType.Table, null, new string[] { "id" }), - Fields: new List { new() { Name = "name", Alias = "geom" } }, - Rest: new(Enabled: true), - GraphQL: new("GeometryAliasPolicy", "GeometryAliasPolicys", Enabled: true), - Permissions: new EntityPermission[] - { - new(Role: "anonymous", - Actions: new EntityAction[] - { - new(Action: EntityActionOperation.Read, - Fields: null, - Policy: new(Request: null, Database: "@item.geom ne null")) - }) - }, - Relationships: null, - Mappings: null); - - await SetUpSingleEntityMetadataProviderAsync("GeometryAliasPolicy", entity); - - await _sqlMetadataProvider.InitializeAsync(); - - Assert.IsTrue( - _sqlMetadataProvider.GetEntityNamesAndDbObjects().ContainsKey("GeometryAliasPolicy"), - message: "A policy referencing an alias of a supported column must be honored, whatever the alias is named."); - - TestHelper.UnsetAllDABEnvironmentVariables(); - } - - /// - /// Test to validate that a simple view over a table holding an unsupported column keeps the - /// key inference it had before the projection was narrowed. - /// A view has no index of its own, so the catalog holds no key for it; the data adapter - /// resolved one through the underlying table, and describing the projection recovers the - /// same information without reintroducing the columns the projection left out. - /// - [TestMethod, TestCategory(TestCategory.MSSQL)] - public async Task ValidateViewKeyIsInferredWithoutConfiguredKeyFields() - { - DatabaseEngine = TestCategory.MSSQL; - TestHelper.SetupDatabaseEnvironment(DatabaseEngine); - - await SetUpSingleEntityMetadataProviderAsync( - "GeometryAllView", - BuildReadOnlyEntity( - entityName: "GeometryAllView", - databaseObject: "dbo.geometry_all_view", - sourceType: EntitySourceType.View, - keyFields: null)); - - await _sqlMetadataProvider.InitializeAsync(); - - Assert.IsTrue( - _sqlMetadataProvider.GetEntityNamesAndDbObjects().TryGetValue("GeometryAllView", out DatabaseObject databaseObject), - message: "Metadata inference failed for a simple view with no configured key."); - - SourceDefinition sourceDefinition = databaseObject.SourceDefinition; - - CollectionAssert.AreEqual( - new List { "id" }, - sourceDefinition.PrimaryKey, - message: "The key of the underlying table is expected to be inferred for the view."); - Assert.IsTrue( - sourceDefinition.Columns.ContainsKey("name"), - message: "A column with a supported data type is expected in the source definition."); - Assert.IsFalse( - sourceDefinition.Columns.ContainsKey("geom"), - message: "A column whose data type cannot be mapped is not expected in the source definition."); - - TestHelper.UnsetAllDABEnvironmentVariables(); - } - - /// - /// Test to validate that a view whose underlying key has a member the projection cannot - /// express does not load with a partial key. - /// Reporting only the readable member would produce a key that does not identify a row, and - /// a partial key silently matches more than one row on an update or a delete — worse than - /// the failure it would replace. - /// - [TestMethod, TestCategory(TestCategory.MSSQL)] - public async Task ValidateViewOverPartiallyUnsupportedKeyDoesNotInferPartialKey() - { - DatabaseEngine = TestCategory.MSSQL; - TestHelper.SetupDatabaseEnvironment(DatabaseEngine); - - await SetUpSingleEntityMetadataProviderAsync( - "HierarchyIdCompositeView", - BuildReadOnlyEntity( - entityName: "HierarchyIdCompositeView", - databaseObject: "dbo.hierarchyid_composite_view", - sourceType: EntitySourceType.View, - keyFields: null)); - - try - { - await _sqlMetadataProvider.InitializeAsync(); - Assert.Fail("Expected DataApiBuilderException was not thrown for a view whose underlying key cannot be expressed by the projection."); - } - catch (DataApiBuilderException ex) - { - Assert.AreEqual(HttpStatusCode.ServiceUnavailable, ex.StatusCode); - Assert.AreEqual(DataApiBuilderException.SubStatusCodes.ErrorInInitialization, ex.SubStatusCode); - Assert.IsTrue( - ex.Message.Contains("node"), - message: "The error is expected to name the key member the object does not expose, rather than " - + $"report a missing primary key. Actual message: {ex.Message}"); - } - - TestHelper.UnsetAllDABEnvironmentVariables(); - } - - /// - /// Test to validate that a view which does not select the underlying primary key is still - /// keyed through a unique key it does carry whole. - /// Unlike a table, where a key column absent from the projection means it was dropped as - /// unsupported, a view may simply not select the primary key. That is ordinary design, not a - /// key the object cannot express, so the search continues into the unique keys. - /// - [TestMethod, TestCategory(TestCategory.MSSQL)] - public async Task ValidateViewWithoutPrimaryKeyFallsBackToUniqueKey() - { - DatabaseEngine = TestCategory.MSSQL; - TestHelper.SetupDatabaseEnvironment(DatabaseEngine); - - await SetUpSingleEntityMetadataProviderAsync( - "UniqueWithoutPkView", - BuildReadOnlyEntity( - entityName: "UniqueWithoutPkView", - databaseObject: "dbo.unique_without_pk_view", - sourceType: EntitySourceType.View, - keyFields: null)); - - await _sqlMetadataProvider.InitializeAsync(); - - Assert.IsTrue( - _sqlMetadataProvider.GetEntityNamesAndDbObjects().TryGetValue("UniqueWithoutPkView", out DatabaseObject databaseObject), - message: "Metadata inference failed for a view that carries a unique key but not the primary key."); - - SourceDefinition sourceDefinition = databaseObject.SourceDefinition; - - CollectionAssert.AreEqual( - new List { "code" }, - sourceDefinition.PrimaryKey, - message: "The unique key the view carries whole is expected to be the key in effect."); - Assert.IsFalse( - sourceDefinition.Columns.ContainsKey("geom"), - message: "A column whose data type cannot be mapped is not expected in the source definition."); - - TestHelper.UnsetAllDABEnvironmentVariables(); - } - - /// - /// Test to validate that a view built on a join does not have one side's key inferred as its - /// own, even when every column the projection selects resolves to that side. - /// The second object of this view contributes only the unsupported column, so once that - /// column is dropped nothing the projection selects points at it. Browse mode still reports - /// its key column, flagged hidden, which is what identifies the result as a join. A key taken - /// from the first object alone would not identify a row, because the join can duplicate its - /// rows — the same class of defect as a partial key. - /// - [TestMethod, TestCategory(TestCategory.MSSQL)] - public async Task ValidateViewOverJoinDoesNotInferKeyFromOneSide() - { - DatabaseEngine = TestCategory.MSSQL; - TestHelper.SetupDatabaseEnvironment(DatabaseEngine); - - await SetUpSingleEntityMetadataProviderAsync( - "JoinGeometryView", - BuildReadOnlyEntity( - entityName: "JoinGeometryView", - databaseObject: "dbo.join_geometry_view", - sourceType: EntitySourceType.View, - keyFields: null)); - - try - { - await _sqlMetadataProvider.InitializeAsync(); - Assert.Fail("Expected DataApiBuilderException was not thrown for a view built on a join with no configured key."); - } - catch (DataApiBuilderException ex) - { - Assert.AreEqual(HttpStatusCode.ServiceUnavailable, ex.StatusCode); - Assert.AreEqual(DataApiBuilderException.SubStatusCodes.ErrorInInitialization, ex.SubStatusCode); - Assert.IsTrue( - ex.Message.Contains("Primary key not configured"), - message: "A join is out of scope for key inference, so the object is expected to require " - + $"source.key-fields exactly as it did before. Actual message: {ex.Message}"); - } - - TestHelper.UnsetAllDABEnvironmentVariables(); - } - - /// - /// Test to validate that a relationship naming a column left out of the projection fails - /// initialization. - /// The exposed and backing column maps are built from the configuration rather than from - /// the source definition, so the reference keeps resolving after the column is gone and - /// fails per request instead — in MultipleCreateOrderHelper, which indexes - /// by each relationship field. - /// - [TestMethod, TestCategory(TestCategory.MSSQL)] - public async Task ValidateRelationshipFieldOverUnsupportedColumnFailsInitialization() - { - DatabaseEngine = TestCategory.MSSQL; - TestHelper.SetupDatabaseEnvironment(DatabaseEngine); - - Entity sourceEntity = BuildReadOnlyEntity( - entityName: "UniqueKeyGeometrySource", - databaseObject: "dbo.unique_key_geometry_table", - sourceType: EntitySourceType.Table, - keyFields: null) - with - { - Relationships = new Dictionary - { - { - "geometryType", - new EntityRelationship( - Cardinality: Cardinality.One, - TargetEntity: "GeometryTypeTarget", - SourceFields: new[] { "geom" }, - TargetFields: new[] { "id" }, - LinkingObject: null, - LinkingSourceFields: Array.Empty(), - LinkingTargetFields: Array.Empty()) - } - } - }; - - await SetUpEntitiesMetadataProviderAsync(new Dictionary - { - { "UniqueKeyGeometrySource", sourceEntity }, - { - "GeometryTypeTarget", - BuildReadOnlyEntity( - entityName: "GeometryTypeTarget", - databaseObject: "dbo.geometry_type_table", - sourceType: EntitySourceType.Table, - keyFields: null) - } - }); - - try - { - await _sqlMetadataProvider.InitializeAsync(); - Assert.Fail("Expected DataApiBuilderException was not thrown for a relationship naming a column whose data type is not supported."); - } - catch (DataApiBuilderException ex) - { - Assert.AreEqual(HttpStatusCode.ServiceUnavailable, ex.StatusCode); - Assert.AreEqual(DataApiBuilderException.SubStatusCodes.ErrorInInitialization, ex.SubStatusCode); - Assert.IsTrue( - ex.Message.Contains("geom"), - message: "The error is expected to name the column the relationship cannot be honored over. " - + $"Actual message: {ex.Message}"); - } - - TestHelper.UnsetAllDABEnvironmentVariables(); - } - - /// - /// Test to validate that a view built on a self-join does not have the object's key - /// inferred as its own. - /// Browse mode reports the physical table rather than the alias, so both instances of the - /// object look like one and the single-source check cannot see the join. What gives it away - /// is the key column being reported twice, once per instance. A self-join duplicates rows - /// of the object just as any other join does, so its key does not identify a row of the - /// result. - /// - [TestMethod, TestCategory(TestCategory.MSSQL)] - public async Task ValidateViewOverSelfJoinDoesNotInferKey() - { - DatabaseEngine = TestCategory.MSSQL; - TestHelper.SetupDatabaseEnvironment(DatabaseEngine); - - await SetUpSingleEntityMetadataProviderAsync( - "SelfJoinGeometryView", - BuildReadOnlyEntity( - entityName: "SelfJoinGeometryView", - databaseObject: "dbo.self_join_geometry_view", - sourceType: EntitySourceType.View, - keyFields: null)); - - try - { - await _sqlMetadataProvider.InitializeAsync(); - Assert.Fail("Expected DataApiBuilderException was not thrown for a view built on a self-join with no configured key."); - } - catch (DataApiBuilderException ex) - { - Assert.AreEqual(HttpStatusCode.ServiceUnavailable, ex.StatusCode); - Assert.AreEqual(DataApiBuilderException.SubStatusCodes.ErrorInInitialization, ex.SubStatusCode); - Assert.IsTrue( - ex.Message.Contains("Primary key not configured"), - message: "A self-join is out of scope for key inference, so the object is expected to require " - + $"source.key-fields exactly as it did before. Actual message: {ex.Message}"); - } - - TestHelper.UnsetAllDABEnvironmentVariables(); - } - - /// - /// Test to validate that a configured key loads a view whose underlying key the projection - /// cannot express. - /// The schema read runs for every object, so it also finds that no key of the object - /// underneath is reachable here. That finding must not reject an object whose - /// source.key-fields already says how to reach it — failing then would recommend precisely - /// what was configured. - /// - [TestMethod, TestCategory(TestCategory.MSSQL)] - public async Task ValidateConfiguredKeyIsHonoredWhenViewOmitsUnderlyingKey() - { - DatabaseEngine = TestCategory.MSSQL; - TestHelper.SetupDatabaseEnvironment(DatabaseEngine); - - await SetUpSingleEntityMetadataProviderAsync( - "HierarchyIdCompositeViewWithKey", - BuildReadOnlyEntity( - entityName: "HierarchyIdCompositeViewWithKey", - databaseObject: "dbo.hierarchyid_composite_view", - sourceType: EntitySourceType.View, - keyFields: new[] { "tenant_id" })); - - await _sqlMetadataProvider.InitializeAsync(); - - Assert.IsTrue( - _sqlMetadataProvider.GetEntityNamesAndDbObjects().TryGetValue("HierarchyIdCompositeViewWithKey", out DatabaseObject databaseObject), - message: "Metadata inference failed for a view whose key is configured through source.key-fields."); - - SourceDefinition sourceDefinition = databaseObject.SourceDefinition; - - CollectionAssert.AreEqual( - new List { "tenant_id" }, - sourceDefinition.PrimaryKey, - message: "The configured key is expected to be the key in effect."); - Assert.IsFalse( - sourceDefinition.Columns.ContainsKey("node"), - message: "A column whose data type cannot be mapped is not expected in the source definition."); - - TestHelper.UnsetAllDABEnvironmentVariables(); - } - - /// - /// Test to validate that a doubled quote inside a policy literal is read as an escaped quote - /// and does not end the literal, so the field prefix that follows stays data. - /// - [TestMethod, TestCategory(TestCategory.MSSQL)] - public async Task ValidateEscapedQuoteInDatabasePolicyLiteralIsNotAFieldReference() - { - DatabaseEngine = TestCategory.MSSQL; - TestHelper.SetupDatabaseEnvironment(DatabaseEngine); - - Entity entity = new( - Source: new("dbo.geometry_type_table", EntitySourceType.Table, null, new string[] { "id" }), - Fields: null, - Rest: new(Enabled: true), - GraphQL: new("GeometryEscapedPolicy", "GeometryEscapedPolicys", Enabled: true), - Permissions: new EntityPermission[] - { - new(Role: "anonymous", - Actions: new EntityAction[] - { - new(Action: EntityActionOperation.Read, - Fields: null, - Policy: new(Request: null, Database: "@item.name ne 'it''s @item.geom'")) - }) - }, - Relationships: null, - Mappings: null); - - await SetUpSingleEntityMetadataProviderAsync("GeometryEscapedPolicy", entity); - - await _sqlMetadataProvider.InitializeAsync(); - - Assert.IsTrue( - _sqlMetadataProvider.GetEntityNamesAndDbObjects().ContainsKey("GeometryEscapedPolicy"), - message: "A doubled quote inside a literal must not end it, so the prefix that follows is not a reference."); - - TestHelper.UnsetAllDABEnvironmentVariables(); - } - - /// - /// Test to validate that the "fields" alias format is rejected when it names a column left - /// out of the projection, the same way "mappings" is. - /// - [TestMethod, TestCategory(TestCategory.MSSQL)] - public async Task ValidateFieldsAliasOverUnsupportedColumnFailsInitialization() - { - DatabaseEngine = TestCategory.MSSQL; - TestHelper.SetupDatabaseEnvironment(DatabaseEngine); - - Entity entity = new( - Source: new("dbo.geometry_type_table", EntitySourceType.Table, null, new string[] { "id" }), - Fields: new List { new() { Name = "geom", Alias = "Position" } }, - Rest: new(Enabled: true), - GraphQL: new("GeometryFieldsAlias", "GeometryFieldsAliass", Enabled: true), - Permissions: new EntityPermission[] - { - new(Role: "anonymous", - Actions: new EntityAction[] { new(Action: EntityActionOperation.Read, Fields: null, Policy: null) }) - }, - Relationships: null, - Mappings: null); - - await SetUpSingleEntityMetadataProviderAsync("GeometryFieldsAlias", entity); - - try - { - await _sqlMetadataProvider.InitializeAsync(); - Assert.Fail("Expected DataApiBuilderException was not thrown for a fields alias over a column of an unsupported data type."); - } - catch (DataApiBuilderException ex) - { - Assert.IsTrue( - ex.Message.Contains("geom") && ex.Message.Contains("fields"), - message: $"The error is expected to name the column and the configuration section. Actual message: {ex.Message}"); - } - - TestHelper.UnsetAllDABEnvironmentVariables(); - } - - /// - /// Test to validate that a permission's "fields.include" naming a column left out of the - /// projection fails initialization. - /// - [TestMethod, TestCategory(TestCategory.MSSQL)] - public async Task ValidateIncludedFieldOverUnsupportedColumnFailsInitialization() - { - DatabaseEngine = TestCategory.MSSQL; - TestHelper.SetupDatabaseEnvironment(DatabaseEngine); - - Entity entity = new( - Source: new("dbo.geometry_type_table", EntitySourceType.Table, null, new string[] { "id" }), - Fields: null, - Rest: new(Enabled: true), - GraphQL: new("GeometryIncluded", "GeometryIncludeds", Enabled: true), - Permissions: new EntityPermission[] - { - new(Role: "anonymous", - Actions: new EntityAction[] - { - new(Action: EntityActionOperation.Read, - Fields: new EntityActionFields( - Exclude: new HashSet(), - Include: new HashSet { "id", "name", "geom" }), - Policy: null) - }) - }, - Relationships: null, - Mappings: null); - - await SetUpSingleEntityMetadataProviderAsync("GeometryIncluded", entity); - - try - { - await _sqlMetadataProvider.InitializeAsync(); - Assert.Fail("Expected DataApiBuilderException was not thrown for a fields.include naming a column of an unsupported data type."); - } - catch (DataApiBuilderException ex) - { - Assert.IsTrue( - ex.Message.Contains("geom") && ex.Message.Contains("permissions"), - message: $"The error is expected to name the column and the configuration section. Actual message: {ex.Message}"); - } - - TestHelper.UnsetAllDABEnvironmentVariables(); - } - - /// - /// Builds a metadata provider over a single in-memory entity and resets the database state. - /// The objects exercised by the unsupported-data-type tests are declared in memory rather - /// than in dab-config.MsSql.json, because several of them fail by design and every MSSQL - /// fixture initializes every configured entity. - /// - private static Task SetUpSingleEntityMetadataProviderAsync(string entityName, Entity entity) - { - return SetUpEntitiesMetadataProviderAsync(new Dictionary { { entityName, entity } }); - } - - /// - /// Sets up a metadata provider over the given entities alone, declared in memory. Needed - /// wherever more than one entity is involved, such as a configured relationship. - /// - private static async Task SetUpEntitiesMetadataProviderAsync(Dictionary entities) - { - RuntimeConfig runtimeConfig = SqlTestHelper.SetupRuntimeConfig() - with - { Entities = new RuntimeEntities(entities) }; - RuntimeConfigProvider runtimeConfigProvider = TestHelper.GenerateInMemoryRuntimeConfigProvider(runtimeConfig); - SetUpSQLMetadataProvider(runtimeConfigProvider); - await ResetDbStateAsync(); - } - - /// - /// Builds a read-only entity over a database object, with optional configured key fields and - /// mappings. - /// - private static Entity BuildReadOnlyEntity( - string entityName, - string databaseObject, - EntitySourceType sourceType, - string[] keyFields, - Dictionary mappings = null) - { - return new Entity( - Source: new(databaseObject, sourceType, null, keyFields), - Fields: null, - Rest: new(Enabled: true), - GraphQL: new(entityName, $"{entityName}s", Enabled: true), - Permissions: new EntityPermission[] - { - new(Role: "anonymous", - Actions: new EntityAction[] { new(Action: EntityActionOperation.Read, Fields: null, Policy: null) }) - }, - Relationships: null, - Mappings: mappings); - } - /// /// Test to validate successful inference of relationship data based on data provided in the config and the metadata /// collected from the MySql database. diff --git a/src/Service.Tests/UnitTests/SqlMetadataProviderUnsupportedColumnTypeTests.cs b/src/Service.Tests/UnitTests/SqlMetadataProviderUnsupportedColumnTypeTests.cs new file mode 100644 index 0000000000..a05bf105de --- /dev/null +++ b/src/Service.Tests/UnitTests/SqlMetadataProviderUnsupportedColumnTypeTests.cs @@ -0,0 +1,298 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Threading.Tasks; +using Azure.DataApiBuilder.Config.DatabasePrimitives; +using Azure.DataApiBuilder.Config.ObjectModel; +using Azure.DataApiBuilder.Service.Exceptions; +using Azure.DataApiBuilder.Service.Tests.SqlTests; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace Azure.DataApiBuilder.Service.Tests.UnitTests +{ + /// + /// Schema discovery over MSSQL objects with columns the data provider has no CLR type for, such as geometry + /// and hierarchyid. Those columns are skipped and the rest of the object is inferred as the data adapter would. + /// Entities are declared in memory because several fixtures in DatabaseSchema-MsSql.sql fail by design. + /// + [TestClass, TestCategory(TestCategory.MSSQL)] + public class SqlMetadataProviderUnsupportedColumnTypeTests : SqlTestBase + { + private const string ENTITY_NAME = "TestEntity"; + private const string RELATED_ENTITY_NAME = "Related"; + + [ClassInitialize] + public static async Task SetupAsync(TestContext context) + { + DatabaseEngine = TestCategory.MSSQL; + TestHelper.SetupDatabaseEnvironment(DatabaseEngine); + SetUpSQLMetadataProvider(TestHelper.GenerateInMemoryRuntimeConfigProvider(SqlTestHelper.SetupRuntimeConfig())); + await ResetDbStateAsync(); + } + + [TestInitialize] + public void TestInitialize() + { + DatabaseEngine = TestCategory.MSSQL; + TestHelper.SetupDatabaseEnvironment(DatabaseEngine); + } + + [TestCleanup] + public void TestCleanup() + { + TestHelper.UnsetAllDABEnvironmentVariables(); + } + + [TestMethod] + public async Task UnsupportedColumnIsSkipped() + { + SourceDefinition definition = await InferSourceDefinitionAsync(CreateEntity("dbo.geometry_type_table", EntitySourceType.Table)); + + CollectionAssert.AreEqual(new[] { "id", "name" }, definition.Columns.Keys.ToArray()); + CollectionAssert.AreEqual(new[] { "id" }, definition.PrimaryKey); + Assert.IsTrue(definition.Columns["id"].IsAutoGenerated, "The identity column should stay auto-generated."); + Assert.IsFalse(definition.Columns["name"].IsNullable, "Nullability should match the database."); + } + + [TestMethod] + public async Task IdentityColumnKeepsItsTypeWhenColumnsAreSkipped() + { + SourceDefinition definition = await InferSourceDefinitionAsync( + CreateEntity("dbo.decimal_identity_geometry_table", EntitySourceType.Table)); + + Assert.AreEqual(typeof(decimal), definition.Columns["id"].SystemType); + Assert.IsTrue(definition.Columns["id"].IsAutoGenerated); + } + + [TestMethod] + public async Task HiddenPeriodColumnsAreNotInferred() + { + SourceDefinition definition = await InferSourceDefinitionAsync( + CreateEntity("dbo.temporal_geometry_type_table", EntitySourceType.Table)); + + CollectionAssert.AreEqual(new[] { "id", "name" }, definition.Columns.Keys.ToArray()); + } + + [TestMethod] + public async Task ColumnReadableUnderLegacyTypeSystemIsKept() + { + SourceDefinition definition = await InferSourceDefinitionAsync( + CreateEntity("dbo.geometry_type_table", EntitySourceType.Table), + connectionStringOptions: "Type System Version=SQL Server 2005;"); + + Assert.AreEqual(typeof(byte[]), definition.Columns["geom"].SystemType); + } + + [DataTestMethod] + [DataRow("dbo.unique_key_geometry_table", EntitySourceType.Table, "code", DisplayName = "Unique key of a table without a primary key")] + [DataRow("dbo.geometry_all_view", EntitySourceType.View, "id", DisplayName = "View that selects the underlying primary key")] + [DataRow("dbo.join_geometry_view", EntitySourceType.View, "id", DisplayName = "View over a join")] + public async Task KeyIsInferredAsTheDataAdapterWould(string databaseObject, EntitySourceType sourceType, string expectedKey) + { + SourceDefinition definition = await InferSourceDefinitionAsync(CreateEntity(databaseObject, sourceType)); + + CollectionAssert.AreEqual(new[] { expectedKey }, definition.PrimaryKey); + } + + [DataTestMethod] + [DataRow("dbo.hierarchyid_unique_table", EntitySourceType.Table, "id", DisplayName = "Table whose unique key is unsupported")] + [DataRow("dbo.hierarchyid_composite_view", EntitySourceType.View, "tenant_id", DisplayName = "View whose key is partly unsupported")] + [DataRow("dbo.unique_without_pk_view", EntitySourceType.View, "code", DisplayName = "View that omits the underlying primary key")] + public async Task ConfiguredKeyIsUsed(string databaseObject, EntitySourceType sourceType, string keyField) + { + SourceDefinition definition = await InferSourceDefinitionAsync( + CreateEntity(databaseObject, sourceType, keyFields: new[] { keyField })); + + CollectionAssert.AreEqual(new[] { keyField }, definition.PrimaryKey); + } + + [DataTestMethod] + [DataRow("dbo.hierarchyid_pk_table", EntitySourceType.Table, DisplayName = "Primary key")] + [DataRow("dbo.hierarchyid_composite_pk_table", EntitySourceType.Table, DisplayName = "Member of a composite primary key")] + [DataRow("dbo.hierarchyid_unique_table", EntitySourceType.Table, DisplayName = "Unique key of a table without a primary key")] + [DataRow("dbo.hierarchyid_composite_view", EntitySourceType.View, DisplayName = "Member of a view's key")] + public async Task UnsupportedKeyColumnFailsInitialization(string databaseObject, EntitySourceType sourceType) + { + DataApiBuilderException exception = await InitializeExpectingFailureAsync(CreateEntity(databaseObject, sourceType)); + + StringAssert.Contains(exception.Message, "has no usable primary key: key column(s) node (hierarchyid)"); + } + + [DataTestMethod] + [DataRow("dbo.unique_without_pk_view", DisplayName = "View that omits the underlying primary key")] + [DataRow("dbo.self_join_geometry_view", DisplayName = "View over a self-join")] + public async Task ViewWithoutInferableKeyRequiresKeyFields(string databaseObject) + { + DataApiBuilderException exception = await InitializeExpectingFailureAsync(CreateEntity(databaseObject, EntitySourceType.View)); + + StringAssert.Contains(exception.Message, "Primary key not configured"); + } + + [TestMethod] + public async Task ObjectWithOnlyUnsupportedColumnsFailsInitialization() + { + DataApiBuilderException exception = await InitializeExpectingFailureAsync( + CreateEntity("dbo.geometry_only_view", EntitySourceType.View, keyFields: new[] { "geom" })); + + StringAssert.Contains(exception.Message, "every column has a data type that is not supported: geom (geometry)"); + } + + [DataTestMethod] + [DataRow("source.key-fields", "the primary key")] + [DataRow("mappings", "mappings")] + [DataRow("fields", "fields")] + [DataRow("fields.include", "fields.include of role 'anonymous'")] + [DataRow("database policy", "the database policy of role 'anonymous'")] + [DataRow("source.fields", "source.fields of relationship 'related'")] + [DataRow("source.fields with a matching alias", "source.fields of relationship 'related'")] + [DataRow("target.fields", "target.fields of relationship 'related'")] + public async Task ReferenceToUnsupportedColumnFailsInitialization(string reference, string expectedSection) + { + Entity entity = CreateEntity("dbo.geometry_type_table", EntitySourceType.Table); + entity = reference switch + { + "source.key-fields" => entity with { Source = entity.Source with { KeyFields = new[] { "geom" } } }, + "mappings" => entity with { Mappings = new() { { "geom", "location" } } }, + "fields" => entity with { Fields = new() { new FieldMetadata { Name = "geom", Alias = "location" } } }, + "fields.include" => entity with { Permissions = CreatePermissions(fields: new(Exclude: new(), Include: new() { "id", "geom" })) }, + "database policy" => entity with { Permissions = CreatePermissions(policy: new(Database: "@item.geom ne null")) }, + "source.fields" => entity with { Relationships = CreateRelationship(sourceField: "geom", targetField: "code") }, + // Relationship fields name backing columns, so an alias with the same name doesn't make the reference valid. + "source.fields with a matching alias" => entity with + { + Fields = new() { new FieldMetadata { Name = "name", Alias = "geom" } }, + Relationships = CreateRelationship(sourceField: "geom", targetField: "code") + }, + "target.fields" => entity with { Relationships = CreateRelationship(sourceField: "name", targetField: "geom") }, + _ => throw new ArgumentOutOfRangeException(nameof(reference)) + }; + + DataApiBuilderException exception = await InitializeEntitiesExpectingFailureAsync(new() + { + { ENTITY_NAME, entity }, + { RELATED_ENTITY_NAME, CreateEntity("dbo.unique_key_geometry_table", EntitySourceType.Table, name: RELATED_ENTITY_NAME) } + }); + + StringAssert.Contains( + exception.Message, + $"Entity '{ENTITY_NAME}': {expectedSection} references column 'geom', whose data type 'geometry' is not supported."); + } + + [DataTestMethod] + [DataRow("@item.name ne '@item.geom'", null, DisplayName = "Reference inside a string literal")] + [DataRow("@item.name ne 'it''s @item.geom'", null, DisplayName = "Reference after an escaped quote inside a string literal")] + [DataRow("@item.geom ne 'x'", "geom", DisplayName = "Alias of a supported column")] + public async Task PolicyNotReferencingUnsupportedColumnIsAccepted(string policy, string nameAlias) + { + Entity entity = CreateEntity("dbo.geometry_type_table", EntitySourceType.Table) with + { + Fields = nameAlias is null ? null : new() { new FieldMetadata { Name = "name", Alias = nameAlias } }, + Permissions = CreatePermissions(policy: new(Database: policy)) + }; + + SourceDefinition definition = await InferSourceDefinitionAsync(entity); + + Assert.IsFalse(definition.Columns.ContainsKey("geom")); + } + + [TestMethod] + public async Task EntitiesNamingTheSameObjectInDifferentCaseShareItsMetadata() + { + Entity lower = CreateEntity("dbo.geometry_type_table", EntitySourceType.Table, name: "Lower"); + Entity upper = CreateEntity("dbo.GEOMETRY_TYPE_TABLE", EntitySourceType.Table, name: "Upper"); + + await InitializeMetadataProviderAsync(new() { { "Lower", lower }, { "Upper", upper } }); + Assert.IsTrue(_sqlMetadataProvider.GetSourceDefinition("Upper").Columns["id"].IsAutoGenerated); + + DataApiBuilderException exception = await InitializeEntitiesExpectingFailureAsync(new() + { + { "Lower", lower }, + { "Upper", upper with { Mappings = new() { { "geom", "location" } } } } + }); + StringAssert.Contains(exception.Message, "Entity 'Upper': mappings references column 'geom'"); + } + + private static Entity CreateEntity( + string databaseObject, + EntitySourceType sourceType, + string[] keyFields = null, + string name = ENTITY_NAME) + { + return new( + Source: new(databaseObject, sourceType, Parameters: null, KeyFields: keyFields), + GraphQL: new(name, $"{name}s"), + Fields: null, + Rest: new(), + Permissions: CreatePermissions(), + Mappings: null, + Relationships: null); + } + + private static EntityPermission[] CreatePermissions(EntityActionFields fields = null, EntityActionPolicy policy = null) + { + return new[] { new EntityPermission("anonymous", new[] { new EntityAction(EntityActionOperation.Read, fields, policy) }) }; + } + + private static Dictionary CreateRelationship(string sourceField, string targetField) + { + return new() + { + { + "related", + new EntityRelationship( + Cardinality: Cardinality.One, + TargetEntity: RELATED_ENTITY_NAME, + SourceFields: new[] { sourceField }, + TargetFields: new[] { targetField }, + LinkingObject: null, + LinkingSourceFields: Array.Empty(), + LinkingTargetFields: Array.Empty()) + } + }; + } + + private static async Task InferSourceDefinitionAsync(Entity entity, string connectionStringOptions = null) + { + await InitializeMetadataProviderAsync(new() { { ENTITY_NAME, entity } }, connectionStringOptions); + return _sqlMetadataProvider.GetSourceDefinition(ENTITY_NAME); + } + + private static Task InitializeExpectingFailureAsync(Entity entity) + { + return InitializeEntitiesExpectingFailureAsync(new() { { ENTITY_NAME, entity } }); + } + + private static async Task InitializeEntitiesExpectingFailureAsync(Dictionary entities) + { + DataApiBuilderException exception = await Assert.ThrowsExceptionAsync( + () => InitializeMetadataProviderAsync(entities)); + + Assert.AreEqual(HttpStatusCode.ServiceUnavailable, exception.StatusCode); + Assert.AreEqual(DataApiBuilderException.SubStatusCodes.ErrorInInitialization, exception.SubStatusCode); + return exception; + } + + private static async Task InitializeMetadataProviderAsync(Dictionary entities, string connectionStringOptions = null) + { + RuntimeConfig runtimeConfig = SqlTestHelper.SetupRuntimeConfig(); + string connectionString = runtimeConfig.DataSource.ConnectionString; + if (connectionStringOptions is not null) + { + connectionString = $"{connectionString.TrimEnd().TrimEnd(';')};{connectionStringOptions}"; + } + + runtimeConfig = runtimeConfig with + { + DataSource = runtimeConfig.DataSource with { ConnectionString = connectionString }, + Entities = new RuntimeEntities(entities) + }; + + SetUpSQLMetadataProvider(TestHelper.GenerateInMemoryRuntimeConfigProvider(runtimeConfig)); + await _sqlMetadataProvider.InitializeAsync(); + } + } +} From 46d3863dcb3666a64ec19036e464a7ced336e5ed Mon Sep 17 00:00:00 2001 From: Aaron Burtle Date: Fri, 13 Nov 2026 11:37:19 -0800 Subject: [PATCH 19/19] Generate autoentities for tables with skipped columns and extend test coverage Autoentities skipped every table with a geometry, geography or hierarchyid column. Schema discovery now leaves such columns out, so a table is only skipped when one of them is in its primary key. The check covers any CLR type and is defined once for both the filter and contains_invalid_types. New tests cover an autoentity, and entities whose fields.include or fields.exclude leave the column out, over REST and GraphQL; a linking object with a skipped column; and linking fields that reference one. --- src/Core/Resolvers/MsSqlQueryBuilder.cs | 58 +++++----- .../Configuration/ConfigurationTests.cs | 103 ++++++++++++++++++ src/Service.Tests/DatabaseSchema-MsSql.sql | 9 ++ ...adataProviderUnsupportedColumnTypeTests.cs | 69 ++++++++++++ 4 files changed, 207 insertions(+), 32 deletions(-) diff --git a/src/Core/Resolvers/MsSqlQueryBuilder.cs b/src/Core/Resolvers/MsSqlQueryBuilder.cs index de2b7071ca..885f030b4f 100644 --- a/src/Core/Resolvers/MsSqlQueryBuilder.cs +++ b/src/Core/Resolvers/MsSqlQueryBuilder.cs @@ -686,6 +686,30 @@ END AS is_system_object FROM all_tables AS o JOIN sys.objects AS so ON so.object_id = o.object_id + ), + tables_with_invalid_types AS + ( + SELECT c.object_id + FROM sys.columns AS c + JOIN sys.types AS ty + ON c.user_type_id = ty.user_type_id + WHERE ty.name IN (N'sql_variant', N'rowversion') + -- Columns of CLR types such as geometry are skipped, which only fails when they are in the primary key. + OR + ( + ty.is_assembly_type = 1 + AND EXISTS + ( + SELECT 1 + FROM sys.index_columns AS ic + JOIN sys.indexes AS i + ON i.object_id = ic.object_id + AND i.index_id = ic.index_id + WHERE ic.object_id = c.object_id + AND ic.column_id = c.column_id + AND i.is_primary_key = 1 + ) + ) ) SELECT a.schema_name AS [schema], @@ -698,22 +722,7 @@ ELSE REPLACE( ) END AS entity_name, CASE - WHEN EXISTS - ( - SELECT 1 - FROM sys.columns AS c - JOIN sys.types AS ty - ON c.user_type_id = ty.user_type_id - WHERE c.object_id = a.object_id - AND ty.name IN - ( - N'geography', - N'geometry', - N'hierarchyid', - N'sql_variant', - N'rowversion' - ) - ) THEN 1 + WHEN a.object_id IN (SELECT object_id FROM tables_with_invalid_types) THEN 1 ELSE 0 END AS contains_invalid_types FROM eligible_tables AS a @@ -742,22 +751,7 @@ WHERE a.full_name LIKE ip.pattern COLLATE DATABASE_DEFAULT ESCAPE '\' AND ( @exclude_invalid_types = 0 - OR NOT EXISTS - ( - SELECT 1 - FROM sys.columns AS c - JOIN sys.types AS ty - ON c.user_type_id = ty.user_type_id - WHERE c.object_id = a.object_id - AND ty.name IN - ( - N'geography', - N'geometry', - N'hierarchyid', - N'sql_variant', - N'rowversion' - ) - ) + OR a.object_id NOT IN (SELECT object_id FROM tables_with_invalid_types) ) ORDER BY a.schema_name, diff --git a/src/Service.Tests/Configuration/ConfigurationTests.cs b/src/Service.Tests/Configuration/ConfigurationTests.cs index b4feb57479..4c72213ab0 100644 --- a/src/Service.Tests/Configuration/ConfigurationTests.cs +++ b/src/Service.Tests/Configuration/ConfigurationTests.cs @@ -6142,6 +6142,109 @@ public async Task TestAutoentitiesAreGeneratedIntoEntities(bool useEntities, int } } + /// + /// A table with a geometry column, which schema discovery skips, is served through an autoentity and through + /// entities whose field permissions leave the column out. + /// + [TestCategory(TestCategory.MSSQL)] + [DataTestMethod] + [DataRow("autoentity", DisplayName = "Autoentity")] + [DataRow("fields.exclude", DisplayName = "Entity whose permissions exclude the column")] + [DataRow("fields.include", DisplayName = "Entity whose permissions include only the other columns")] + public async Task TestTableWithUnsupportedColumnTypeIsServed(string entityKind) + { + // Arrange + const string ENTITY_NAME = "dbo_geometry_type_table"; + EntityActionFields fields = entityKind switch + { + "fields.exclude" => new(Exclude: new() { "geom" }), + "fields.include" => new(Exclude: new(), Include: new() { "id", "name" }), + _ => null + }; + EntityPermission[] permissions = new[] + { + new EntityPermission(AuthorizationResolver.ROLE_ANONYMOUS, new[] { new EntityAction(EntityActionOperation.Read, fields, new()) }) + }; + + Dictionary entityMap = new(); + Dictionary autoentityMap = new(); + if (entityKind == "autoentity") + { + autoentityMap.Add("GeometryAutoEntity", new Autoentity( + Patterns: new AutoentityPatterns(Include: new[] { "dbo.geometry_type_table" }, Exclude: null, Name: null), + Template: new AutoentityTemplate( + Rest: new EntityRestOptions(Enabled: true), + GraphQL: new EntityGraphQLOptions(Singular: string.Empty, Plural: string.Empty, Enabled: true), + Health: null, + Cache: null), + Permissions: permissions)); + } + else + { + entityMap.Add(ENTITY_NAME, new Entity( + Source: new("dbo.geometry_type_table", EntitySourceType.Table, null, null), + Fields: null, + Rest: new(Enabled: true), + GraphQL: new(Singular: string.Empty, Plural: string.Empty), + Permissions: permissions, + Relationships: null, + Mappings: null)); + } + + DataSource dataSource = new(DatabaseType.MSSQL, + GetConnectionStringFromEnvironmentConfig(environment: TestCategory.MSSQL), Options: null); + + RuntimeConfig configuration = new( + Schema: "TestUnsupportedColumnTypeSchema", + DataSource: dataSource, + Runtime: new( + Rest: new(Enabled: true), + GraphQL: new(Enabled: true), + Mcp: new(Enabled: false), + Host: new( + Cors: null, + Authentication: new Config.ObjectModel.AuthenticationOptions( + Provider: nameof(EasyAuthType.StaticWebApps), + Jwt: null + ) + ) + ), + Entities: new(entityMap), + Autoentities: autoentityMap.Count > 0 ? new RuntimeAutoentities(autoentityMap) : null + ); + + File.WriteAllText(CUSTOM_CONFIG_FILENAME, configuration.ToJson()); + + string[] args = new[] { $"--ConfigFileName={CUSTOM_CONFIG_FILENAME}" }; + + using (TestServer server = new(Program.CreateWebHostBuilder(args))) + using (HttpClient client = server.CreateClient()) + { + // Act + using HttpRequestMessage restRequest = new(HttpMethod.Get, $"/api/{ENTITY_NAME}"); + using HttpResponseMessage restResponse = await client.SendAsync(restRequest); + + object graphqlPayload = new { query = $"{{ {ENTITY_NAME}s {{ items {{ id name }} }} }}" }; + using HttpRequestMessage graphqlRequest = new(HttpMethod.Post, "/graphql") + { + Content = JsonContent.Create(graphqlPayload) + }; + using HttpResponseMessage graphqlResponse = await client.SendAsync(graphqlRequest); + + // Assert + const string EXPECTED_ROW = @"{""id"":1,""name"":""point""}"; + string restResponseBody = await restResponse.Content.ReadAsStringAsync(); + Assert.AreEqual(HttpStatusCode.OK, restResponse.StatusCode, restResponseBody); + StringAssert.Contains(restResponseBody, EXPECTED_ROW); + Assert.IsFalse(restResponseBody.Contains(@"""geom"""), "The skipped column should not be returned."); + + string graphqlResponseBody = await graphqlResponse.Content.ReadAsStringAsync(); + Assert.AreEqual(HttpStatusCode.OK, graphqlResponse.StatusCode, graphqlResponseBody); + Assert.IsFalse(graphqlResponseBody.Contains("errors"), graphqlResponseBody); + StringAssert.Contains(graphqlResponseBody, EXPECTED_ROW); + } + } + /// /// This test validates that multiple autoentities with the same object name /// but different schemas can be generated and accessed properly with the diff --git a/src/Service.Tests/DatabaseSchema-MsSql.sql b/src/Service.Tests/DatabaseSchema-MsSql.sql index 59e8df345a..d0bf3e1f72 100644 --- a/src/Service.Tests/DatabaseSchema-MsSql.sql +++ b/src/Service.Tests/DatabaseSchema-MsSql.sql @@ -58,6 +58,7 @@ DROP TABLE IF EXISTS hierarchyid_unique_table; DROP TABLE IF EXISTS unique_key_geometry_table; DROP TABLE IF EXISTS decimal_identity_geometry_table; DROP TABLE IF EXISTS pk_and_unique_geometry_table; +DROP TABLE IF EXISTS geometry_linking_table; -- System versioning has to be released before the temporal table can be dropped. IF OBJECT_ID('dbo.temporal_geometry_type_table', 'U') IS NOT NULL AND OBJECTPROPERTY(OBJECT_ID('dbo.temporal_geometry_type_table'), 'TableTemporalType') = 2 @@ -336,6 +337,14 @@ CREATE TABLE decimal_identity_geometry_table( geom geometry NULL ); +-- A linking table with an unsupported column, for relationships that go through it. +CREATE TABLE geometry_linking_table( + geometry_id int NOT NULL, + code varchar(20) NOT NULL, + geom geometry NULL, + CONSTRAINT PK_geometry_linking_table PRIMARY KEY (geometry_id, code) +); + CREATE TABLE profiles( id int IDENTITY(5001, 1) PRIMARY KEY, metadata json NULL diff --git a/src/Service.Tests/UnitTests/SqlMetadataProviderUnsupportedColumnTypeTests.cs b/src/Service.Tests/UnitTests/SqlMetadataProviderUnsupportedColumnTypeTests.cs index a05bf105de..ade450b92b 100644 --- a/src/Service.Tests/UnitTests/SqlMetadataProviderUnsupportedColumnTypeTests.cs +++ b/src/Service.Tests/UnitTests/SqlMetadataProviderUnsupportedColumnTypeTests.cs @@ -5,9 +5,11 @@ using System.Collections.Generic; using System.Linq; using System.Net; +using System.Text.Json.Nodes; using System.Threading.Tasks; using Azure.DataApiBuilder.Config.DatabasePrimitives; using Azure.DataApiBuilder.Config.ObjectModel; +using Azure.DataApiBuilder.Core.Services; using Azure.DataApiBuilder.Service.Exceptions; using Azure.DataApiBuilder.Service.Tests.SqlTests; using Microsoft.VisualStudio.TestTools.UnitTesting; @@ -150,6 +152,8 @@ public async Task ObjectWithOnlyUnsupportedColumnsFailsInitialization() [DataRow("source.fields", "source.fields of relationship 'related'")] [DataRow("source.fields with a matching alias", "source.fields of relationship 'related'")] [DataRow("target.fields", "target.fields of relationship 'related'")] + [DataRow("linking.source.fields", "linking.source.fields of relationship 'related'")] + [DataRow("linking.target.fields", "linking.target.fields of relationship 'related'")] public async Task ReferenceToUnsupportedColumnFailsInitialization(string reference, string expectedSection) { Entity entity = CreateEntity("dbo.geometry_type_table", EntitySourceType.Table); @@ -168,6 +172,8 @@ public async Task ReferenceToUnsupportedColumnFailsInitialization(string referen Relationships = CreateRelationship(sourceField: "geom", targetField: "code") }, "target.fields" => entity with { Relationships = CreateRelationship(sourceField: "name", targetField: "geom") }, + "linking.source.fields" => entity with { Relationships = CreateLinkingRelationship(linkingSourceField: "geom", linkingTargetField: "code") }, + "linking.target.fields" => entity with { Relationships = CreateLinkingRelationship(linkingSourceField: "geometry_id", linkingTargetField: "geom") }, _ => throw new ArgumentOutOfRangeException(nameof(reference)) }; @@ -216,6 +222,51 @@ public async Task EntitiesNamingTheSameObjectInDifferentCaseShareItsMetadata() StringAssert.Contains(exception.Message, "Entity 'Upper': mappings references column 'geom'"); } + [TestMethod] + public async Task LinkingObjectWithUnsupportedColumnIsLoaded() + { + Entity entity = CreateEntity("dbo.geometry_type_table", EntitySourceType.Table) with + { + Relationships = CreateLinkingRelationship(linkingSourceField: "geometry_id", linkingTargetField: "code") + }; + + await InitializeMetadataProviderAsync(new() + { + { ENTITY_NAME, entity }, + { RELATED_ENTITY_NAME, CreateEntity("dbo.unique_key_geometry_table", EntitySourceType.Table, name: RELATED_ENTITY_NAME) } + }); + + string linkingEntityName = _sqlMetadataProvider.GetLinkingEntities().Keys.Single(); + CollectionAssert.AreEqual( + new[] { "geometry_id", "code" }, + _sqlMetadataProvider.GetSourceDefinition(linkingEntityName).Columns.Keys.ToArray()); + } + + [TestMethod] + public async Task AutoentitiesExcludeOnlyTablesWithAnUnsupportedPrimaryKeyColumn() + { + Autoentity autoentity = new( + new AutoentityPatterns( + Include: new[] + { + "dbo.geometry_type_table", + "dbo.decimal_identity_geometry_table", + "dbo.hierarchyid_pk_table", + "dbo.hierarchyid_composite_pk_table" + }, + Exclude: null, + Name: "{object}"), + Template: null, + Permissions: null); + SetUpSQLMetadataProvider(TestHelper.GenerateInMemoryRuntimeConfigProvider(SqlTestHelper.SetupRuntimeConfig())); + + JsonArray tables = await ((MsSqlMetadataProvider)_sqlMetadataProvider).QueryAutoentitiesAsync("autoentity", autoentity); + + CollectionAssert.AreEqual( + new[] { "decimal_identity_geometry_table", "geometry_type_table" }, + tables.Select(table => table["object"].ToString()).ToArray()); + } + private static Entity CreateEntity( string databaseObject, EntitySourceType sourceType, @@ -255,6 +306,24 @@ private static Dictionary CreateRelationship(string }; } + private static Dictionary CreateLinkingRelationship(string linkingSourceField, string linkingTargetField) + { + return new() + { + { + "related", + new EntityRelationship( + Cardinality: Cardinality.Many, + TargetEntity: RELATED_ENTITY_NAME, + SourceFields: new[] { "id" }, + TargetFields: new[] { "code" }, + LinkingObject: "dbo.geometry_linking_table", + LinkingSourceFields: new[] { linkingSourceField }, + LinkingTargetFields: new[] { linkingTargetField }) + } + }; + } + private static async Task InferSourceDefinitionAsync(Entity entity, string connectionStringOptions = null) { await InitializeMetadataProviderAsync(new() { { ENTITY_NAME, entity } }, connectionStringOptions);