diff --git a/src/ServiceControl.Audit.Persistence.RavenDB/CustomChecks/CheckRavenDBSearchEngine.cs b/src/ServiceControl.Audit.Persistence.RavenDB/CustomChecks/CheckRavenDBSearchEngine.cs new file mode 100644 index 0000000000..1196e4578d --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.RavenDB/CustomChecks/CheckRavenDBSearchEngine.cs @@ -0,0 +1,22 @@ +namespace ServiceControl.Audit.Persistence.RavenDB.CustomChecks; + +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using NServiceBus.CustomChecks; +using ServiceControl.RavenDB; + +class CheckRavenDBSearchEngine(IRavenDocumentStoreProvider documentStoreProvider, DatabaseConfiguration databaseConfiguration) : CustomCheck("Audit Database Search Engine", "ServiceControl.Audit Health", TimeSpan.FromHours(1)) +{ + public override async Task PerformCheck(CancellationToken cancellationToken = default) + { + var documentStore = await documentStoreProvider.GetDocumentStore(cancellationToken); + + var coraxIndexes = await StartupChecks.FindIndexesUsingCorax(documentStore, databaseConfiguration.Name, cancellationToken); + + return coraxIndexes.Length == 0 + ? CheckResult.Pass + : CheckResult.Failed(StartupChecks.CoraxIndexesMessage(coraxIndexes.Select(i => $"{databaseConfiguration.Name}/{i}"))); + } +} diff --git a/src/ServiceControl.Audit.Persistence.RavenDB/DatabaseSetup.cs b/src/ServiceControl.Audit.Persistence.RavenDB/DatabaseSetup.cs index 701f7647bc..2560fc128e 100644 --- a/src/ServiceControl.Audit.Persistence.RavenDB/DatabaseSetup.cs +++ b/src/ServiceControl.Audit.Persistence.RavenDB/DatabaseSetup.cs @@ -15,6 +15,7 @@ using Raven.Client.ServerWide.Operations; using Raven.Client.ServerWide.Operations.Configuration; using Indexes; +using ServiceControl.RavenDB; using SagaAudit; class DatabaseSetup(DatabaseConfiguration configuration) @@ -27,6 +28,8 @@ public async Task Execute(IDocumentStore documentStore, CancellationToken cancel await CreateIndexes(documentStore, configuration.EnableFullTextSearch, cancellationToken); + await StartupChecks.WarnIfIndexesUseCorax(documentStore, configuration.Name, cancellationToken); + await LicenseStatusCheck.WaitForLicenseOrThrow(documentStore, cancellationToken); await ConfigureExpiration(documentStore, cancellationToken); } @@ -41,7 +44,9 @@ async Task CreateDatabase(IDocumentStore documentStore, string databaseName, Can { var databaseRecord = new DatabaseRecord(databaseName); - SetSearchEngineType(databaseRecord, SearchEngineType.Corax); + // New databases use Lucene: smaller indexes, lower memory usage and faster for our index definitions. + // Existing databases keep the engine they were created with, see UpdateDatabaseSettings. + SetSearchEngineType(databaseRecord, SearchEngineType.Lucene); await documentStore.Maintenance.Server.SendAsync(new CreateDatabaseOperation(databaseRecord), cancellationToken); } @@ -56,6 +61,9 @@ async Task UpdateDatabaseSettings(IDocumentStore documentStore, string databaseN { var databaseRecord = await documentStore.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(databaseName), cancellationToken) ?? throw new InvalidOperationException($"Database '{databaseName}' does not exist."); + // Existing databases keep their configured search engine. Changing it would trigger a full rebuild of all + // indexes, which can take a long time and a lot of resources on large databases. Databases created before the + // search engine was pinned explicitly get Corax, which was the default at the time. if (!SetSearchEngineType(databaseRecord, SearchEngineType.Corax)) { return; diff --git a/src/ServiceControl.Audit.Persistence.RavenDB/RavenPersistence.cs b/src/ServiceControl.Audit.Persistence.RavenDB/RavenPersistence.cs index 89cbe27739..e2c9239cbd 100644 --- a/src/ServiceControl.Audit.Persistence.RavenDB/RavenPersistence.cs +++ b/src/ServiceControl.Audit.Persistence.RavenDB/RavenPersistence.cs @@ -22,6 +22,7 @@ public void AddPersistence(IServiceCollection services) endpointConfiguration.AddCustomCheck(); endpointConfiguration.AddCustomCheck(); endpointConfiguration.AddCustomCheck(); + endpointConfiguration.AddCustomCheck(); } services.AddSingleton(); diff --git a/src/ServiceControl.Audit.Persistence.Tests.RavenDB/ApprovalFiles/CustomCheckTests.VerifyCustomChecks.approved.txt b/src/ServiceControl.Audit.Persistence.Tests.RavenDB/ApprovalFiles/CustomCheckTests.VerifyCustomChecks.approved.txt index f510666c86..88fa262a1e 100644 --- a/src/ServiceControl.Audit.Persistence.Tests.RavenDB/ApprovalFiles/CustomCheckTests.VerifyCustomChecks.approved.txt +++ b/src/ServiceControl.Audit.Persistence.Tests.RavenDB/ApprovalFiles/CustomCheckTests.VerifyCustomChecks.approved.txt @@ -1,4 +1,5 @@ ServiceControl.Audit Health: Audit Database Index Lag +ServiceControl.Audit Health: Audit Database Search Engine ServiceControl.Audit Health: Audit Message Ingestion Process ServiceControl.Audit Health: RavenDB dirty memory Storage space: ServiceControl.Audit database \ No newline at end of file diff --git a/src/ServiceControl.Audit.Persistence.Tests.RavenDB/IndexSetupTests.cs b/src/ServiceControl.Audit.Persistence.Tests.RavenDB/IndexSetupTests.cs index ee655f7fc2..3df471801d 100644 --- a/src/ServiceControl.Audit.Persistence.Tests.RavenDB/IndexSetupTests.cs +++ b/src/ServiceControl.Audit.Persistence.Tests.RavenDB/IndexSetupTests.cs @@ -9,22 +9,43 @@ namespace ServiceControl.Audit.Persistence.Tests; using Raven.Client.Documents.Operations.Indexes; using Raven.Client.Exceptions; using Raven.Client.Exceptions.Documents.Indexes; +using ServiceControl.RavenDB; [TestFixture] class IndexSetupTests : PersistenceTestFixture { [Test] - public async Task Corax_should_be_the_default_search_engine_type() + public async Task Lucene_should_be_the_default_search_engine_type_for_new_databases() { var indexes = await configuration.DocumentStore.Maintenance.SendAsync(new GetIndexesOperation(0, int.MaxValue)); foreach (var index in indexes) { var indexStats = await configuration.DocumentStore.Maintenance.SendAsync(new GetIndexStatisticsOperation(DatabaseSetup.MessagesViewIndexWithFulltextSearchName)); - Assert.That(indexStats.SearchEngineType, Is.EqualTo(SearchEngineType.Corax), $"{index.Name} is not using Corax"); + Assert.That(indexStats.SearchEngineType, Is.EqualTo(SearchEngineType.Lucene), $"{index.Name} is not using Lucene"); } } + [Test] + public async Task Startup_check_should_not_report_corax_indexes_for_new_database() + { + var coraxIndexes = await StartupChecks.FindIndexesUsingCorax(configuration.DocumentStore, configuration.DocumentStore.Database, TestTimeoutCancellationToken); + + Assert.That(coraxIndexes, Is.Empty); + } + + [Test] + public async Task Startup_check_should_report_indexes_using_corax() + { + var index = new MessagesViewIndexWithFullTextSearch { Configuration = { ["Indexing.Static.SearchEngineType"] = SearchEngineType.Corax.ToString() } }; + + await UpdateIndex(index); + + var coraxIndexes = await StartupChecks.FindIndexesUsingCorax(configuration.DocumentStore, configuration.DocumentStore.Database, TestTimeoutCancellationToken); + + Assert.That(coraxIndexes, Is.EqualTo(new[] { index.IndexName })); + } + [Test] public async Task Free_text_search_index_should_be_used_by_default() { @@ -50,11 +71,11 @@ public async Task Free_text_search_index_can_be_opted_out_from() [Test] public async Task Indexes_should_be_reset_on_setup() { - var index = new MessagesViewIndexWithFullTextSearch { Configuration = { ["Indexing.Static.SearchEngineType"] = SearchEngineType.Lucene.ToString() } }; + var index = new MessagesViewIndexWithFullTextSearch { Configuration = { ["Indexing.Static.SearchEngineType"] = SearchEngineType.Corax.ToString() } }; var indexWithCustomConfigStats = await UpdateIndex(index); - Assert.That(indexWithCustomConfigStats.SearchEngineType, Is.EqualTo(SearchEngineType.Lucene)); + Assert.That(indexWithCustomConfigStats.SearchEngineType, Is.EqualTo(SearchEngineType.Corax)); await DatabaseSetup.CreateIndexes(configuration.DocumentStore, true, TestTimeoutCancellationToken); @@ -62,7 +83,7 @@ public async Task Indexes_should_be_reset_on_setup() var indexAfterResetStats = await configuration.DocumentStore.Maintenance.SendAsync(new GetIndexStatisticsOperation(index.IndexName)); - Assert.That(indexAfterResetStats.SearchEngineType, Is.EqualTo(SearchEngineType.Corax)); + Assert.That(indexAfterResetStats.SearchEngineType, Is.EqualTo(SearchEngineType.Lucene)); } [Test] @@ -70,13 +91,13 @@ public async Task Indexes_should_not_be_reset_on_setup_when_locked_as_ignore() { var index = new MessagesViewIndexWithFullTextSearch { - Configuration = { ["Indexing.Static.SearchEngineType"] = SearchEngineType.Lucene.ToString() }, + Configuration = { ["Indexing.Static.SearchEngineType"] = SearchEngineType.Corax.ToString() }, LockMode = IndexLockMode.LockedIgnore }; var indexStatsBefore = await UpdateIndex(index); - Assert.That(indexStatsBefore.SearchEngineType, Is.EqualTo(SearchEngineType.Lucene)); + Assert.That(indexStatsBefore.SearchEngineType, Is.EqualTo(SearchEngineType.Corax)); await DatabaseSetup.CreateIndexes(configuration.DocumentStore, true, TestTimeoutCancellationToken); @@ -84,7 +105,7 @@ public async Task Indexes_should_not_be_reset_on_setup_when_locked_as_ignore() await Task.Delay(1000); var indexStatsAfter = await configuration.DocumentStore.Maintenance.SendAsync(new GetIndexStatisticsOperation(index.IndexName)); - Assert.That(indexStatsAfter.SearchEngineType, Is.EqualTo(SearchEngineType.Lucene)); + Assert.That(indexStatsAfter.SearchEngineType, Is.EqualTo(SearchEngineType.Corax)); } [Test] @@ -92,7 +113,7 @@ public async Task Indexes_should_not_be_reset_on_setup_when_locked_as_error() { var index = new MessagesViewIndexWithFullTextSearch { - Configuration = { ["Indexing.Static.SearchEngineType"] = SearchEngineType.Lucene.ToString() }, + Configuration = { ["Indexing.Static.SearchEngineType"] = SearchEngineType.Corax.ToString() }, LockMode = IndexLockMode.LockedError }; diff --git a/src/ServiceControl.Persistence.RavenDB/CustomChecks/CheckRavenDBSearchEngine.cs b/src/ServiceControl.Persistence.RavenDB/CustomChecks/CheckRavenDBSearchEngine.cs new file mode 100644 index 0000000000..27fd9ebc72 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/CustomChecks/CheckRavenDBSearchEngine.cs @@ -0,0 +1,30 @@ +namespace ServiceControl.Persistence.RavenDB.CustomChecks; + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using NServiceBus.CustomChecks; +using ServiceControl.RavenDB; + +class CheckRavenDBSearchEngine(IRavenDocumentStoreProvider documentStoreProvider, RavenPersisterSettings settings) : CustomCheck("Error Database Search Engine", "ServiceControl Health", TimeSpan.FromHours(1)) +{ + public override async Task PerformCheck(CancellationToken cancellationToken = default) + { + var documentStore = await documentStoreProvider.GetDocumentStore(cancellationToken); + + var coraxIndexes = new List(); + + foreach (var databaseName in new[] { settings.DatabaseName, settings.ThroughputDatabaseName }) + { + foreach (var indexName in await StartupChecks.FindIndexesUsingCorax(documentStore, databaseName, cancellationToken)) + { + coraxIndexes.Add($"{databaseName}/{indexName}"); + } + } + + return coraxIndexes.Count == 0 + ? CheckResult.Pass + : CheckResult.Failed(StartupChecks.CoraxIndexesMessage(coraxIndexes)); + } +} diff --git a/src/ServiceControl.Persistence.RavenDB/DatabaseSetup.cs b/src/ServiceControl.Persistence.RavenDB/DatabaseSetup.cs index 147c2393ac..73c6812e09 100644 --- a/src/ServiceControl.Persistence.RavenDB/DatabaseSetup.cs +++ b/src/ServiceControl.Persistence.RavenDB/DatabaseSetup.cs @@ -10,6 +10,7 @@ namespace ServiceControl.Persistence.RavenDB using Raven.Client.ServerWide; using Raven.Client.ServerWide.Operations; using Raven.Client.ServerWide.Operations.Configuration; + using ServiceControl.RavenDB; class DatabaseSetup(RavenPersisterSettings settings, IDocumentStore documentStore) { @@ -23,6 +24,9 @@ public async Task Execute(CancellationToken cancellationToken = default) await IndexCreation.CreateIndexesAsync(typeof(DatabaseSetup).Assembly, documentStore, null, null, cancellationToken); + await StartupChecks.WarnIfIndexesUseCorax(documentStore, settings.DatabaseName, cancellationToken); + await StartupChecks.WarnIfIndexesUseCorax(documentStore, settings.ThroughputDatabaseName, cancellationToken); + await LicenseStatusCheck.WaitForLicenseOrThrow(documentStore, cancellationToken); await ConfigureExpiration(settings, cancellationToken); } @@ -36,8 +40,11 @@ async Task CreateDatabase(string databaseName, CancellationToken cancellationTok try { var databaseRecord = new DatabaseRecord(databaseName); - databaseRecord.Settings.Add("Indexing.Auto.SearchEngineType", "Corax"); - databaseRecord.Settings.Add("Indexing.Static.SearchEngineType", "Corax"); + + // New databases use Lucene: smaller indexes, lower memory usage and faster for our index definitions. + // Existing databases keep the engine they were created with, see UpdateDatabaseSettings. + databaseRecord.Settings.Add("Indexing.Auto.SearchEngineType", "Lucene"); + databaseRecord.Settings.Add("Indexing.Static.SearchEngineType", "Lucene"); await documentStore.Maintenance.Server.SendAsync(new CreateDatabaseOperation(databaseRecord), cancellationToken); } @@ -57,6 +64,9 @@ async Task UpdateDatabaseSettings(string databaseName, CancellationToken cancell throw new InvalidOperationException($"Database '{databaseName}' does not exist."); } + // Existing databases keep their configured search engine. Changing it would trigger a full rebuild of all + // indexes, which can take a long time and a lot of resources on large databases. Databases created before the + // search engine was pinned explicitly get Corax, which was the default at the time. var updated = false; updated |= dbRecord.Settings.TryAdd("Indexing.Auto.SearchEngineType", "Corax"); diff --git a/src/ServiceControl.Persistence.RavenDB/RavenPersistence.cs b/src/ServiceControl.Persistence.RavenDB/RavenPersistence.cs index 8e82edd6a9..6b75afe71e 100644 --- a/src/ServiceControl.Persistence.RavenDB/RavenPersistence.cs +++ b/src/ServiceControl.Persistence.RavenDB/RavenPersistence.cs @@ -46,6 +46,7 @@ public void AddPersistence(IServiceCollection services) services.AddCustomCheck(); services.AddCustomCheck(); services.AddCustomCheck(); + services.AddCustomCheck(); services.AddSingleton(); services.AddSingleton(); diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/ApprovalFiles/CustomCheckTests.VerifyCustomChecks.approved.txt b/src/ServiceControl.Persistence.Tests.RavenDB/ApprovalFiles/CustomCheckTests.VerifyCustomChecks.approved.txt index ecb6bed984..5f089b7c02 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDB/ApprovalFiles/CustomCheckTests.VerifyCustomChecks.approved.txt +++ b/src/ServiceControl.Persistence.Tests.RavenDB/ApprovalFiles/CustomCheckTests.VerifyCustomChecks.approved.txt @@ -1,5 +1,6 @@ ServiceControl Health: Error Database Index Errors ServiceControl Health: Error Database Index Lag +ServiceControl Health: Error Database Search Engine ServiceControl Health: Message Ingestion Process ServiceControl Health: RavenDB dirty memory Storage space: ServiceControl database \ No newline at end of file diff --git a/src/ServiceControl.RavenDB/StartupChecks.cs b/src/ServiceControl.RavenDB/StartupChecks.cs index e5fb5e92fa..a8c44278db 100644 --- a/src/ServiceControl.RavenDB/StartupChecks.cs +++ b/src/ServiceControl.RavenDB/StartupChecks.cs @@ -1,12 +1,46 @@ namespace ServiceControl.RavenDB { + using System.Collections.Generic; using System.Reflection; using System.Threading; + using Microsoft.Extensions.Logging; using Raven.Client.Documents; + using Raven.Client.Documents.Indexes; + using Raven.Client.Documents.Operations.Indexes; using Raven.Client.ServerWide.Operations; + using ServiceControl.Infrastructure; public static class StartupChecks { + public static async Task WarnIfIndexesUseCorax(IDocumentStore store, string databaseName, CancellationToken cancellationToken = default) + { + // New databases are created with Lucene, existing databases keep whatever search engine they were created + // with as switching would trigger a full rebuild of all indexes. Let the operator know so they can plan the + // transition to Lucene themselves. + var coraxIndexes = await FindIndexesUsingCorax(store, databaseName, cancellationToken); + + if (coraxIndexes.Length > 0) + { + Logger.LogWarning(CoraxIndexesMessage(coraxIndexes.Select(i => $"{databaseName}/{i}"))); + } + } + + public static string CoraxIndexesMessage(IEnumerable indexes) => + $"The following RavenDB index(es) use the Corax search engine: {string.Join(", ", indexes)}. " + + "Lucene indexes are smaller, use less memory and perform better for ServiceControl workloads, and are the default for new databases. " + + "Consider switching these indexes to Lucene. Note that switching triggers a full rebuild of the index: on very large databases this can take days depending on the available compute, " + + "and while the rebuild is running ingestion and indexing rates can be degraded. Plan the switch accordingly."; + + public static async Task FindIndexesUsingCorax(IDocumentStore store, string databaseName, CancellationToken cancellationToken = default) + { + var indexStats = await store.Maintenance.ForDatabase(databaseName).SendAsync(new GetIndexesStatisticsOperation(), cancellationToken); + + return indexStats + .Where(i => i.SearchEngineType == SearchEngineType.Corax) + .Select(i => i.Name) + .ToArray(); + } + public static async Task EnsureServerVersion(IDocumentStore store, CancellationToken cancellationToken = default) { // RavenDB compatibility policy is that the major/minor version of the server must be @@ -31,5 +65,7 @@ public static async Task EnsureServerVersion(IDocumentStore store, CancellationT throw new Exception($"ServiceControl expects RavenDB Server version {clientProductVersion} or higher, but the server is using {serverProductVersion}."); } } + + static readonly ILogger Logger = LoggerUtil.CreateStaticLogger(typeof(StartupChecks)); } }