Use Lucene search engine for new RavenDB databases and flag indexes still on Corax

src/ServiceControl.Audit.Persistence.RavenDB/CustomChecks/CheckRavenDBSearchEngine.cs

@@ -0,0 +1,22 @@
+namespace ServiceControl.Audit.Persistence.RavenDB.CustomChecks;
+
+using System;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using NServiceBus.CustomChecks;
+using ServiceControl.RavenDB;
+
+class CheckRavenDBSearchEngine(IRavenDocumentStoreProvider documentStoreProvider, DatabaseConfiguration databaseConfiguration) : CustomCheck("Audit Database Search Engine", "ServiceControl.Audit Health", TimeSpan.FromHours(1))
+{
+    public override async Task<CheckResult> PerformCheck(CancellationToken cancellationToken = default)
+    {
+        var documentStore = await documentStoreProvider.GetDocumentStore(cancellationToken);
+
+        var coraxIndexes = await StartupChecks.FindIndexesUsingCorax(documentStore, databaseConfiguration.Name, cancellationToken);
+
+        return coraxIndexes.Length == 0
+            ? CheckResult.Pass
+            : CheckResult.Failed(StartupChecks.CoraxIndexesMessage(coraxIndexes.Select(i => $"{databaseConfiguration.Name}/{i}")));
+    }
+}

src/ServiceControl.Audit.Persistence.RavenDB/DatabaseSetup.cs

@@ -15,6 +15,7 @@
 using Raven.Client.ServerWide.Operations;
 using Raven.Client.ServerWide.Operations.Configuration;
 using Indexes;
+using ServiceControl.RavenDB;
 using SagaAudit;
 
 class DatabaseSetup(DatabaseConfiguration configuration)
@@ -27,6 +28,8 @@ public async Task Execute(IDocumentStore documentStore, CancellationToken cancel
 
         await CreateIndexes(documentStore, configuration.EnableFullTextSearch, cancellationToken);
 
+        await StartupChecks.WarnIfIndexesUseCorax(documentStore, configuration.Name, cancellationToken);
+
         await LicenseStatusCheck.WaitForLicenseOrThrow(documentStore, cancellationToken);
         await ConfigureExpiration(documentStore, cancellationToken);
     }
@@ -41,7 +44,9 @@ async Task CreateDatabase(IDocumentStore documentStore, string databaseName, Can
             {
                 var databaseRecord = new DatabaseRecord(databaseName);
 
-                SetSearchEngineType(databaseRecord, SearchEngineType.Corax);
+                // New databases use Lucene: smaller indexes, lower memory usage and faster for our index definitions.
+                // Existing databases keep the engine they were created with, see UpdateDatabaseSettings.
+                SetSearchEngineType(databaseRecord, SearchEngineType.Lucene);
 
                 await documentStore.Maintenance.Server.SendAsync(new CreateDatabaseOperation(databaseRecord), cancellationToken);
             }
@@ -56,6 +61,9 @@ async Task UpdateDatabaseSettings(IDocumentStore documentStore, string databaseN
     {
         var databaseRecord = await documentStore.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(databaseName), cancellationToken) ?? throw new InvalidOperationException($"Database '{databaseName}' does not exist.");
 
+        // Existing databases keep their configured search engine. Changing it would trigger a full rebuild of all
+        // indexes, which can take a long time and a lot of resources on large databases. Databases created before the
+        // search engine was pinned explicitly get Corax, which was the default at the time.
         if (!SetSearchEngineType(databaseRecord, SearchEngineType.Corax))
         {
             return;

src/ServiceControl.Audit.Persistence.RavenDB/RavenPersistence.cs

@@ -22,6 +22,7 @@ public void AddPersistence(IServiceCollection services)
                 endpointConfiguration.AddCustomCheck<CheckFreeDiskSpace>();
                 endpointConfiguration.AddCustomCheck<CheckMinimumStorageRequiredForIngestion>();
                 endpointConfiguration.AddCustomCheck<CheckRavenDBIndexLag>();
+                endpointConfiguration.AddCustomCheck<CheckRavenDBSearchEngine>();
             }
 
             services.AddSingleton<IAuditDataStore, RavenAuditDataStore>();

src/ServiceControl.Audit.Persistence.Tests.RavenDB/ApprovalFiles/CustomCheckTests.VerifyCustomChecks.approved.txt

@@ -1,4 +1,5 @@
 ServiceControl.Audit Health: Audit Database Index Lag
+ServiceControl.Audit Health: Audit Database Search Engine
 ServiceControl.Audit Health: Audit Message Ingestion Process
 ServiceControl.Audit Health: RavenDB dirty memory
 Storage space: ServiceControl.Audit database

src/ServiceControl.Audit.Persistence.Tests.RavenDB/IndexSetupTests.cs

@@ -9,22 +9,43 @@ namespace ServiceControl.Audit.Persistence.Tests;
 using Raven.Client.Documents.Operations.Indexes;
 using Raven.Client.Exceptions;
 using Raven.Client.Exceptions.Documents.Indexes;
+using ServiceControl.RavenDB;
 
 [TestFixture]
 class IndexSetupTests : PersistenceTestFixture
 {
     [Test]
-    public async Task Corax_should_be_the_default_search_engine_type()
+    public async Task Lucene_should_be_the_default_search_engine_type_for_new_databases()
     {
         var indexes = await configuration.DocumentStore.Maintenance.SendAsync(new GetIndexesOperation(0, int.MaxValue));
 
         foreach (var index in indexes)
         {
             var indexStats = await configuration.DocumentStore.Maintenance.SendAsync(new GetIndexStatisticsOperation(DatabaseSetup.MessagesViewIndexWithFulltextSearchName));
-            Assert.That(indexStats.SearchEngineType, Is.EqualTo(SearchEngineType.Corax), $"{index.Name} is not using Corax");
+            Assert.That(indexStats.SearchEngineType, Is.EqualTo(SearchEngineType.Lucene), $"{index.Name} is not using Lucene");
         }
     }
 
+    [Test]
+    public async Task Startup_check_should_not_report_corax_indexes_for_new_database()
+    {
+        var coraxIndexes = await StartupChecks.FindIndexesUsingCorax(configuration.DocumentStore, configuration.DocumentStore.Database, TestTimeoutCancellationToken);
+
+        Assert.That(coraxIndexes, Is.Empty);
+    }
+
+    [Test]
+    public async Task Startup_check_should_report_indexes_using_corax()
+    {
+        var index = new MessagesViewIndexWithFullTextSearch { Configuration = { ["Indexing.Static.SearchEngineType"] = SearchEngineType.Corax.ToString() } };
+
+        await UpdateIndex(index);
+
+        var coraxIndexes = await StartupChecks.FindIndexesUsingCorax(configuration.DocumentStore, configuration.DocumentStore.Database, TestTimeoutCancellationToken);
+
+        Assert.That(coraxIndexes, Is.EqualTo(new[] { index.IndexName }));
+    }
+
     [Test]
     public async Task Free_text_search_index_should_be_used_by_default()
     {
@@ -50,49 +71,49 @@ public async Task Free_text_search_index_can_be_opted_out_from()
     [Test]
     public async Task Indexes_should_be_reset_on_setup()
     {
-        var index = new MessagesViewIndexWithFullTextSearch { Configuration = { ["Indexing.Static.SearchEngineType"] = SearchEngineType.Lucene.ToString() } };
+        var index = new MessagesViewIndexWithFullTextSearch { Configuration = { ["Indexing.Static.SearchEngineType"] = SearchEngineType.Corax.ToString() } };
 
         var indexWithCustomConfigStats = await UpdateIndex(index);
 
-        Assert.That(indexWithCustomConfigStats.SearchEngineType, Is.EqualTo(SearchEngineType.Lucene));
+        Assert.That(indexWithCustomConfigStats.SearchEngineType, Is.EqualTo(SearchEngineType.Corax));
 
         await DatabaseSetup.CreateIndexes(configuration.DocumentStore, true, TestTimeoutCancellationToken);
 
         await WaitForIndexDefinitionUpdate(indexWithCustomConfigStats);
 
         var indexAfterResetStats = await configuration.DocumentStore.Maintenance.SendAsync(new GetIndexStatisticsOperation(index.IndexName));
 
-        Assert.That(indexAfterResetStats.SearchEngineType, Is.EqualTo(SearchEngineType.Corax));
+        Assert.That(indexAfterResetStats.SearchEngineType, Is.EqualTo(SearchEngineType.Lucene));
     }
 
     [Test]
     public async Task Indexes_should_not_be_reset_on_setup_when_locked_as_ignore()
     {
         var index = new MessagesViewIndexWithFullTextSearch
         {
-            Configuration = { ["Indexing.Static.SearchEngineType"] = SearchEngineType.Lucene.ToString() },
+            Configuration = { ["Indexing.Static.SearchEngineType"] = SearchEngineType.Corax.ToString() },
             LockMode = IndexLockMode.LockedIgnore
         };
 
         var indexStatsBefore = await UpdateIndex(index);
 
-        Assert.That(indexStatsBefore.SearchEngineType, Is.EqualTo(SearchEngineType.Lucene));
+        Assert.That(indexStatsBefore.SearchEngineType, Is.EqualTo(SearchEngineType.Corax));
 
         await DatabaseSetup.CreateIndexes(configuration.DocumentStore, true, TestTimeoutCancellationToken);
 
         // raven will ignore the update since index was locked, so best we can do is wait a bit and check that settings hasn't changed
         await Task.Delay(1000);
 
         var indexStatsAfter = await configuration.DocumentStore.Maintenance.SendAsync(new GetIndexStatisticsOperation(index.IndexName));
-        Assert.That(indexStatsAfter.SearchEngineType, Is.EqualTo(SearchEngineType.Lucene));
+        Assert.That(indexStatsAfter.SearchEngineType, Is.EqualTo(SearchEngineType.Corax));
     }
 
     [Test]
     public async Task Indexes_should_not_be_reset_on_setup_when_locked_as_error()
     {
         var index = new MessagesViewIndexWithFullTextSearch
         {
-            Configuration = { ["Indexing.Static.SearchEngineType"] = SearchEngineType.Lucene.ToString() },
+            Configuration = { ["Indexing.Static.SearchEngineType"] = SearchEngineType.Corax.ToString() },
             LockMode = IndexLockMode.LockedError
         };
 

src/ServiceControl.Persistence.RavenDB/CustomChecks/CheckRavenDBSearchEngine.cs

@@ -0,0 +1,30 @@
+namespace ServiceControl.Persistence.RavenDB.CustomChecks;
+
+using System;
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+using NServiceBus.CustomChecks;
+using ServiceControl.RavenDB;
+
+class CheckRavenDBSearchEngine(IRavenDocumentStoreProvider documentStoreProvider, RavenPersisterSettings settings) : CustomCheck("Error Database Search Engine", "ServiceControl Health", TimeSpan.FromHours(1))
+{
+    public override async Task<CheckResult> PerformCheck(CancellationToken cancellationToken = default)
+    {
+        var documentStore = await documentStoreProvider.GetDocumentStore(cancellationToken);
+
+        var coraxIndexes = new List<string>();
+
+        foreach (var databaseName in new[] { settings.DatabaseName, settings.ThroughputDatabaseName })
+        {
+            foreach (var indexName in await StartupChecks.FindIndexesUsingCorax(documentStore, databaseName, cancellationToken))
+            {
+                coraxIndexes.Add($"{databaseName}/{indexName}");
+            }
+        }
+
+        return coraxIndexes.Count == 0
+            ? CheckResult.Pass
+            : CheckResult.Failed(StartupChecks.CoraxIndexesMessage(coraxIndexes));
+    }
+}

src/ServiceControl.Persistence.RavenDB/DatabaseSetup.cs

@@ -10,6 +10,7 @@ namespace ServiceControl.Persistence.RavenDB
     using Raven.Client.ServerWide;
     using Raven.Client.ServerWide.Operations;
     using Raven.Client.ServerWide.Operations.Configuration;
+    using ServiceControl.RavenDB;
 
     class DatabaseSetup(RavenPersisterSettings settings, IDocumentStore documentStore)
     {
@@ -23,6 +24,9 @@ public async Task Execute(CancellationToken cancellationToken = default)
 
             await IndexCreation.CreateIndexesAsync(typeof(DatabaseSetup).Assembly, documentStore, null, null, cancellationToken);
 
+            await StartupChecks.WarnIfIndexesUseCorax(documentStore, settings.DatabaseName, cancellationToken);
+            await StartupChecks.WarnIfIndexesUseCorax(documentStore, settings.ThroughputDatabaseName, cancellationToken);
+
             await LicenseStatusCheck.WaitForLicenseOrThrow(documentStore, cancellationToken);
             await ConfigureExpiration(settings, cancellationToken);
         }
@@ -36,8 +40,11 @@ async Task CreateDatabase(string databaseName, CancellationToken cancellationTok
                 try
                 {
                     var databaseRecord = new DatabaseRecord(databaseName);
-                    databaseRecord.Settings.Add("Indexing.Auto.SearchEngineType", "Corax");
-                    databaseRecord.Settings.Add("Indexing.Static.SearchEngineType", "Corax");
+
+                    // New databases use Lucene: smaller indexes, lower memory usage and faster for our index definitions.
+                    // Existing databases keep the engine they were created with, see UpdateDatabaseSettings.
+                    databaseRecord.Settings.Add("Indexing.Auto.SearchEngineType", "Lucene");
+                    databaseRecord.Settings.Add("Indexing.Static.SearchEngineType", "Lucene");
 
                     await documentStore.Maintenance.Server.SendAsync(new CreateDatabaseOperation(databaseRecord), cancellationToken);
                 }
@@ -57,6 +64,9 @@ async Task UpdateDatabaseSettings(string databaseName, CancellationToken cancell
                 throw new InvalidOperationException($"Database '{databaseName}' does not exist.");
             }
 
+            // Existing databases keep their configured search engine. Changing it would trigger a full rebuild of all
+            // indexes, which can take a long time and a lot of resources on large databases. Databases created before the
+            // search engine was pinned explicitly get Corax, which was the default at the time.
             var updated = false;
 
             updated |= dbRecord.Settings.TryAdd("Indexing.Auto.SearchEngineType", "Corax");

src/ServiceControl.Persistence.RavenDB/RavenPersistence.cs

@@ -46,6 +46,7 @@ public void AddPersistence(IServiceCollection services)
         services.AddCustomCheck<CheckFreeDiskSpace>();
         services.AddCustomCheck<CheckMinimumStorageRequiredForIngestion>();
         services.AddCustomCheck<CheckDirtyMemory>();
+        services.AddCustomCheck<CheckRavenDBSearchEngine>();
 
         services.AddSingleton<MemoryInformationRetriever>();
         services.AddSingleton<OperationsManager>();

src/ServiceControl.Persistence.Tests.RavenDB/ApprovalFiles/CustomCheckTests.VerifyCustomChecks.approved.txt

@@ -1,5 +1,6 @@
 ServiceControl Health: Error Database Index Errors
 ServiceControl Health: Error Database Index Lag
+ServiceControl Health: Error Database Search Engine
 ServiceControl Health: Message Ingestion Process
 ServiceControl Health: RavenDB dirty memory
 Storage space: ServiceControl database

src/ServiceControl.RavenDB/StartupChecks.cs

@@ -1,12 +1,46 @@
 namespace ServiceControl.RavenDB
 {
+    using System.Collections.Generic;
     using System.Reflection;
     using System.Threading;
+    using Microsoft.Extensions.Logging;
     using Raven.Client.Documents;
+    using Raven.Client.Documents.Indexes;
+    using Raven.Client.Documents.Operations.Indexes;
     using Raven.Client.ServerWide.Operations;
+    using ServiceControl.Infrastructure;
 
     public static class StartupChecks
     {
+        public static async Task WarnIfIndexesUseCorax(IDocumentStore store, string databaseName, CancellationToken cancellationToken = default)
+        {
+            // New databases are created with Lucene, existing databases keep whatever search engine they were created
+            // with as switching would trigger a full rebuild of all indexes. Let the operator know so they can plan the
+            // transition to Lucene themselves.
+            var coraxIndexes = await FindIndexesUsingCorax(store, databaseName, cancellationToken);
+
+            if (coraxIndexes.Length > 0)
+            {
+                Logger.LogWarning(CoraxIndexesMessage(coraxIndexes.Select(i => $"{databaseName}/{i}")));
+            }
+        }
+
+        public static string CoraxIndexesMessage(IEnumerable<string> indexes) =>
+            $"The following RavenDB index(es) use the Corax search engine: {string.Join(", ", indexes)}. " +
+            "Lucene indexes are smaller, use less memory and perform better for ServiceControl workloads, and are the default for new databases. " +
+            "Consider switching these indexes to Lucene. Note that switching triggers a full rebuild of the index: on very large databases this can take days depending on the available compute, " +
+            "and while the rebuild is running ingestion and indexing rates can be degraded. Plan the switch accordingly.";
+
+        public static async Task<string[]> FindIndexesUsingCorax(IDocumentStore store, string databaseName, CancellationToken cancellationToken = default)
+        {
+            var indexStats = await store.Maintenance.ForDatabase(databaseName).SendAsync(new GetIndexesStatisticsOperation(), cancellationToken);
+
+            return indexStats
+                .Where(i => i.SearchEngineType == SearchEngineType.Corax)
+                .Select(i => i.Name)
+                .ToArray();
+        }
+
         public static async Task EnsureServerVersion(IDocumentStore store, CancellationToken cancellationToken = default)
         {
             // RavenDB compatibility policy is that the major/minor version of the server must be
@@ -31,5 +65,7 @@ public static async Task EnsureServerVersion(IDocumentStore store, CancellationT
                 throw new Exception($"ServiceControl expects RavenDB Server version {clientProductVersion} or higher, but the server is using {serverProductVersion}.");
             }
         }
+
+        static readonly ILogger Logger = LoggerUtil.CreateStaticLogger(typeof(StartupChecks));
     }
 }