From 603c4aab7cc177f4c970d82939e564bfd299663e Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Fri, 28 Aug 2026 16:56:44 +0200 Subject: [PATCH 1/2] =?UTF-8?q?=E2=9C=A8=20Exclude=20temporal,=20numeric?= =?UTF-8?q?=20and=20boolean=20values=20from=20audit=20full-text=20search?= =?UTF-8?q?=20index?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Dates, durations, sizes and booleans in message metadata and headers were being tokenized into the Query search field of both MessagesView indexes. Nobody searches for them, and because they are unique per message they inflate the term dictionary far more than the shared tokens (endpoint names, message types) do. Identifiers are intentionally kept, since /messages/search/{id} is a full-text search. Duplicate values (e.g. metadata MessageId vs NServiceBus.MessageId header) are now indexed once. The analyzer is referenced by its short name since the index rebuilds anyway. --- .../Indexes/MessagesViewIndex.cs | 29 +++++++++++------ .../MessagesViewIndexWithFullTextSearch.cs | 31 +++++++++++++------ 2 files changed, 41 insertions(+), 19 deletions(-) diff --git a/src/ServiceControl.Audit.Persistence.RavenDB/Indexes/MessagesViewIndex.cs b/src/ServiceControl.Audit.Persistence.RavenDB/Indexes/MessagesViewIndex.cs index fa9a3bc75f..8af20f8ac4 100644 --- a/src/ServiceControl.Audit.Persistence.RavenDB/Indexes/MessagesViewIndex.cs +++ b/src/ServiceControl.Audit.Persistence.RavenDB/Indexes/MessagesViewIndex.cs @@ -25,20 +25,31 @@ from message in messages CriticalTime = (TimeSpan?)message.MessageMetadata["CriticalTime"], ProcessingTime = (TimeSpan?)message.MessageMetadata["ProcessingTime"], DeliveryTime = (TimeSpan?)message.MessageMetadata["DeliveryTime"], - Query = message.MessageMetadata.Select(_ => _.Value.ToString()).Union(new[] - { - string.Join(" ", message.Headers.Select(x => x.Value)) - }).ToArray(), + // Dates, durations, sizes and booleans only add tokens nobody searches for, so they are excluded. + // Identifiers (message/conversation/correlation ids) are kept: /messages/search/{id} relies on them. + Query = message.MessageMetadata + .Where(m => m.Key != "TimeSent" && m.Key != "CriticalTime" && m.Key != "ProcessingTime" + && m.Key != "DeliveryTime" && m.Key != "ContentLength" && m.Key != "BodyUrl" + && m.Key != "IsSystemMessage" && m.Key != "IsRetried" && m.Key != "BodyNotStored" + && m.Key != "OriginatesFromSaga") + .Select(m => m.Value.ToString()) + .Concat(message.Headers + .Where(h => h.Key != "NServiceBus.TimeSent" && h.Key != "NServiceBus.ProcessingStarted" && h.Key != "NServiceBus.ProcessingEnded" + && h.Key != "NServiceBus.DeliverAt" && h.Key != "NServiceBus.Timeout.Expire" && h.Key != "NServiceBus.Retries.Timestamp" + && h.Key != "NServiceBus.ExceptionInfo.TimeOfFailure" && h.Key != "NServiceBus.TimeOfFailure" && h.Key != "NServiceBus.NonDurableMessage" + && h.Key != "NServiceBus.TimeToBeReceived") + .Select(h => h.Value)) + .Where(v => v != null && v.Length > 0) + .Distinct() + .ToArray(), ConversationId = (string)message.MessageMetadata["ConversationId"] }; Index(x => x.Query, FieldIndexing.Search); - // Not using typeof() to prevent dependency on Lucene. - // Unfortunately while "StandardAnalyzer" would probably be better and more future-proof here, - // we can't change this string without causing any existing audit database to completely rebuild this index. - // If this index *must* be changed for some other reason, the analyzer name should be changed at the same time. - Analyze(x => x.Query, "Lucene.Net.Analysis.Standard.StandardAnalyzer, Lucene.Net, Version=3.0.3.0, Culture=neutral, PublicKeyToken=85089178b9ac3181"); + // Any change to this index definition (map or analyzer) causes existing audit databases to rebuild the index on startup. + // The analyzer name deliberately does not use typeof() to prevent a dependency on Lucene. + Analyze(x => x.Query, "StandardAnalyzer"); } public class SortAndFilterOptions diff --git a/src/ServiceControl.Audit.Persistence.RavenDB/Indexes/MessagesViewIndexWithFullTextSearch.cs b/src/ServiceControl.Audit.Persistence.RavenDB/Indexes/MessagesViewIndexWithFullTextSearch.cs index 1dca30993f..a7de29e212 100644 --- a/src/ServiceControl.Audit.Persistence.RavenDB/Indexes/MessagesViewIndexWithFullTextSearch.cs +++ b/src/ServiceControl.Audit.Persistence.RavenDB/Indexes/MessagesViewIndexWithFullTextSearch.cs @@ -25,21 +25,32 @@ from message in messages CriticalTime = (TimeSpan?)message.MessageMetadata["CriticalTime"], ProcessingTime = (TimeSpan?)message.MessageMetadata["ProcessingTime"], DeliveryTime = (TimeSpan?)message.MessageMetadata["DeliveryTime"], - Query = message.MessageMetadata.Select(_ => _.Value.ToString()).Union(new[] - { - string.Join(" ", message.Headers.Select(x => x.Value)), - LoadAttachment(message, "body").GetContentAsString() - }).ToArray(), + // Dates, durations, sizes and booleans only add tokens nobody searches for, so they are excluded. + // Identifiers (message/conversation/correlation ids) are kept: /messages/search/{id} relies on them. + Query = message.MessageMetadata + .Where(m => m.Key != "TimeSent" && m.Key != "CriticalTime" && m.Key != "ProcessingTime" + && m.Key != "DeliveryTime" && m.Key != "ContentLength" && m.Key != "BodyUrl" + && m.Key != "IsSystemMessage" && m.Key != "IsRetried" && m.Key != "BodyNotStored" + && m.Key != "OriginatesFromSaga") + .Select(m => m.Value.ToString()) + .Concat(message.Headers + .Where(h => h.Key != "NServiceBus.TimeSent" && h.Key != "NServiceBus.ProcessingStarted" && h.Key != "NServiceBus.ProcessingEnded" + && h.Key != "NServiceBus.DeliverAt" && h.Key != "NServiceBus.Timeout.Expire" && h.Key != "NServiceBus.Retries.Timestamp" + && h.Key != "NServiceBus.ExceptionInfo.TimeOfFailure" && h.Key != "NServiceBus.TimeOfFailure" && h.Key != "NServiceBus.NonDurableMessage" + && h.Key != "NServiceBus.TimeToBeReceived") + .Select(h => h.Value)) + .Where(v => v != null && v.Length > 0) + .Distinct() + .Concat(new[] { LoadAttachment(message, "body").GetContentAsString() }) + .ToArray(), ConversationId = (string)message.MessageMetadata["ConversationId"] }; Index(x => x.Query, FieldIndexing.Search); - // Not using typeof() to prevent dependency on Lucene. - // Unfortunately while "StandardAnalyzer" would probably be better and more future-proof here, - // we can't change this string without causing any existing audit database to completely rebuild this index. - // If this index *must* be changed for some other reason, the analyzer name should be changed at the same time. - Analyze(x => x.Query, "Lucene.Net.Analysis.Standard.StandardAnalyzer, Lucene.Net, Version=3.0.3.0, Culture=neutral, PublicKeyToken=85089178b9ac3181"); + // Any change to this index definition (map or analyzer) causes existing audit databases to rebuild the index on startup. + // The analyzer name deliberately does not use typeof() to prevent a dependency on Lucene. + Analyze(x => x.Query, "StandardAnalyzer"); } } } \ No newline at end of file From 77633567a6c06368ed58e842751936312c158f4b Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Fri, 28 Aug 2026 17:08:29 +0200 Subject: [PATCH 2/2] =?UTF-8?q?=E2=9C=A8=20Match=20message=20and=20convers?= =?UTF-8?q?ation=20ids=20exactly=20instead=20of=20via=20full-text=20search?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Identifiers are never partially matched, so the audit search now ORs an exact match on the MessageId and ConversationId index fields with the full-text search, and the ids (and their header copies) are dropped from the Query field. The OR group is built with the DocumentQuery API because the LINQ provider does not parenthesize it, which would bind the endpoint/time-range filters to the last OR term only. --- ...ssed_message_searched_by_conversationid.cs | 59 +++++++++++++++++++ .../Indexes/MessagesViewIndex.cs | 9 ++- .../MessagesViewIndexWithFullTextSearch.cs | 9 ++- .../RavenAuditDataStore.cs | 20 +++++-- 4 files changed, 87 insertions(+), 10 deletions(-) create mode 100644 src/ServiceControl.Audit.AcceptanceTests/Auditing/When_processed_message_searched_by_conversationid.cs diff --git a/src/ServiceControl.Audit.AcceptanceTests/Auditing/When_processed_message_searched_by_conversationid.cs b/src/ServiceControl.Audit.AcceptanceTests/Auditing/When_processed_message_searched_by_conversationid.cs new file mode 100644 index 0000000000..20a15256a8 --- /dev/null +++ b/src/ServiceControl.Audit.AcceptanceTests/Auditing/When_processed_message_searched_by_conversationid.cs @@ -0,0 +1,59 @@ +namespace ServiceControl.Audit.AcceptanceTests.Auditing +{ + using System.Threading.Tasks; + using AcceptanceTesting; + using AcceptanceTesting.EndpointTemplates; + using Audit.Auditing.MessagesView; + using NServiceBus; + using NServiceBus.AcceptanceTesting; + using NServiceBus.AcceptanceTesting.Customization; + using NServiceBus.Settings; + using NUnit.Framework; + + class When_processed_message_searched_by_conversationid : AcceptanceTest + { + [Test] + public async Task Should_be_found() => + await Define() + .WithEndpoint(b => b.When((bus, c) => bus.Send(new MyMessage()))) + .WithEndpoint() + .Done(async c => c.ConversationId != null && await this.TryGetMany("/api/messages/search/" + c.ConversationId)) + .Run(); + + public class Sender : EndpointConfigurationBuilder + { + public Sender() => + EndpointSetup(c => + { + var routing = c.ConfigureRouting(); + routing.RouteToEndpoint(typeof(MyMessage), typeof(Receiver)); + }); + } + + public class Receiver : EndpointConfigurationBuilder + { + public Receiver() => EndpointSetup(); + + [Handler] + public class MyMessageHandler(MyContext testContext, IReadOnlySettings settings) + : IHandleMessages + { + public Task Handle(MyMessage message, IMessageHandlerContext context) + { + testContext.EndpointNameOfReceivingEndpoint = settings.EndpointName(); + testContext.ConversationId = context.MessageHeaders[Headers.ConversationId]; + return Task.CompletedTask; + } + } + } + + public class MyMessage : ICommand; + + public class MyContext : ScenarioContext + { + public string ConversationId { get; set; } + + public string EndpointNameOfReceivingEndpoint { get; set; } + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Audit.Persistence.RavenDB/Indexes/MessagesViewIndex.cs b/src/ServiceControl.Audit.Persistence.RavenDB/Indexes/MessagesViewIndex.cs index 8af20f8ac4..5765bb94a2 100644 --- a/src/ServiceControl.Audit.Persistence.RavenDB/Indexes/MessagesViewIndex.cs +++ b/src/ServiceControl.Audit.Persistence.RavenDB/Indexes/MessagesViewIndex.cs @@ -26,15 +26,18 @@ from message in messages ProcessingTime = (TimeSpan?)message.MessageMetadata["ProcessingTime"], DeliveryTime = (TimeSpan?)message.MessageMetadata["DeliveryTime"], // Dates, durations, sizes and booleans only add tokens nobody searches for, so they are excluded. - // Identifiers (message/conversation/correlation ids) are kept: /messages/search/{id} relies on them. + // Identifiers are excluded too: they are matched exactly on the MessageId/ConversationId fields instead. Query = message.MessageMetadata - .Where(m => m.Key != "TimeSent" && m.Key != "CriticalTime" && m.Key != "ProcessingTime" + .Where(m => m.Key != "MessageId" && m.Key != "ConversationId" && m.Key != "RelatedToId" + && m.Key != "TimeSent" && m.Key != "CriticalTime" && m.Key != "ProcessingTime" && m.Key != "DeliveryTime" && m.Key != "ContentLength" && m.Key != "BodyUrl" && m.Key != "IsSystemMessage" && m.Key != "IsRetried" && m.Key != "BodyNotStored" && m.Key != "OriginatesFromSaga") .Select(m => m.Value.ToString()) .Concat(message.Headers - .Where(h => h.Key != "NServiceBus.TimeSent" && h.Key != "NServiceBus.ProcessingStarted" && h.Key != "NServiceBus.ProcessingEnded" + .Where(h => h.Key != "NServiceBus.MessageId" && h.Key != "NServiceBus.ConversationId" + && h.Key != "NServiceBus.CorrelationId" && h.Key != "NServiceBus.RelatedTo" + && h.Key != "NServiceBus.TimeSent" && h.Key != "NServiceBus.ProcessingStarted" && h.Key != "NServiceBus.ProcessingEnded" && h.Key != "NServiceBus.DeliverAt" && h.Key != "NServiceBus.Timeout.Expire" && h.Key != "NServiceBus.Retries.Timestamp" && h.Key != "NServiceBus.ExceptionInfo.TimeOfFailure" && h.Key != "NServiceBus.TimeOfFailure" && h.Key != "NServiceBus.NonDurableMessage" && h.Key != "NServiceBus.TimeToBeReceived") diff --git a/src/ServiceControl.Audit.Persistence.RavenDB/Indexes/MessagesViewIndexWithFullTextSearch.cs b/src/ServiceControl.Audit.Persistence.RavenDB/Indexes/MessagesViewIndexWithFullTextSearch.cs index a7de29e212..d4f224e8da 100644 --- a/src/ServiceControl.Audit.Persistence.RavenDB/Indexes/MessagesViewIndexWithFullTextSearch.cs +++ b/src/ServiceControl.Audit.Persistence.RavenDB/Indexes/MessagesViewIndexWithFullTextSearch.cs @@ -26,15 +26,18 @@ from message in messages ProcessingTime = (TimeSpan?)message.MessageMetadata["ProcessingTime"], DeliveryTime = (TimeSpan?)message.MessageMetadata["DeliveryTime"], // Dates, durations, sizes and booleans only add tokens nobody searches for, so they are excluded. - // Identifiers (message/conversation/correlation ids) are kept: /messages/search/{id} relies on them. + // Identifiers are excluded too: they are matched exactly on the MessageId/ConversationId fields instead. Query = message.MessageMetadata - .Where(m => m.Key != "TimeSent" && m.Key != "CriticalTime" && m.Key != "ProcessingTime" + .Where(m => m.Key != "MessageId" && m.Key != "ConversationId" && m.Key != "RelatedToId" + && m.Key != "TimeSent" && m.Key != "CriticalTime" && m.Key != "ProcessingTime" && m.Key != "DeliveryTime" && m.Key != "ContentLength" && m.Key != "BodyUrl" && m.Key != "IsSystemMessage" && m.Key != "IsRetried" && m.Key != "BodyNotStored" && m.Key != "OriginatesFromSaga") .Select(m => m.Value.ToString()) .Concat(message.Headers - .Where(h => h.Key != "NServiceBus.TimeSent" && h.Key != "NServiceBus.ProcessingStarted" && h.Key != "NServiceBus.ProcessingEnded" + .Where(h => h.Key != "NServiceBus.MessageId" && h.Key != "NServiceBus.ConversationId" + && h.Key != "NServiceBus.CorrelationId" && h.Key != "NServiceBus.RelatedTo" + && h.Key != "NServiceBus.TimeSent" && h.Key != "NServiceBus.ProcessingStarted" && h.Key != "NServiceBus.ProcessingEnded" && h.Key != "NServiceBus.DeliverAt" && h.Key != "NServiceBus.Timeout.Expire" && h.Key != "NServiceBus.Retries.Timestamp" && h.Key != "NServiceBus.ExceptionInfo.TimeOfFailure" && h.Key != "NServiceBus.TimeOfFailure" && h.Key != "NServiceBus.NonDurableMessage" && h.Key != "NServiceBus.TimeToBeReceived") diff --git a/src/ServiceControl.Audit.Persistence.RavenDB/RavenAuditDataStore.cs b/src/ServiceControl.Audit.Persistence.RavenDB/RavenAuditDataStore.cs index 56ab14f906..844d7f42f2 100644 --- a/src/ServiceControl.Audit.Persistence.RavenDB/RavenAuditDataStore.cs +++ b/src/ServiceControl.Audit.Persistence.RavenDB/RavenAuditDataStore.cs @@ -9,6 +9,8 @@ using Extensions; using Indexes; using Raven.Client.Documents; + using Raven.Client.Documents.Linq; + using Raven.Client.Documents.Session; using ServiceControl.Audit.Auditing; using ServiceControl.Audit.Infrastructure; using ServiceControl.SagaAudit; @@ -46,9 +48,8 @@ public async Task>> GetMessages(bool includeSyst public async Task>> QueryMessages(string searchParam, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange timeSentRange, CancellationToken cancellationToken = default) { using var session = await sessionProvider.OpenSession(cancellationToken: cancellationToken); - var results = await session.Query(GetIndexName(isFullTextSearchEnabled)) + var results = await SearchMessages(session, searchParam) .Statistics(out var stats) - .Search(x => x.Query, searchParam) .FilterBySentTimeRange(timeSentRange) .Sort(sortInfo) .Paging(pagingInfo) @@ -61,9 +62,8 @@ public async Task>> QueryMessages(string searchP public async Task>> QueryMessagesByReceivingEndpointAndKeyword(string endpoint, string keyword, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange timeSentRange, CancellationToken cancellationToken = default) { using var session = await sessionProvider.OpenSession(cancellationToken: cancellationToken); - var results = await session.Query(GetIndexName(isFullTextSearchEnabled)) + var results = await SearchMessages(session, keyword) .Statistics(out var stats) - .Search(x => x.Query, keyword) .Where(m => m.ReceivingEndpointName == endpoint) .FilterBySentTimeRange(timeSentRange) .Sort(sortInfo) @@ -74,6 +74,18 @@ public async Task>> QueryMessagesByReceivingEndp return new QueryResult>(results, stats.ToQueryStatsInfo()); } + // Identifiers are not part of the full-text Query field; they are matched exactly on their own fields. + // Built as a DocumentQuery because the LINQ provider does not parenthesize an OR group, so subsequent Where + // clauses would bind to the last OR term only. + IRavenQueryable SearchMessages(IAsyncDocumentSession session, string searchParam) => + session.Advanced.AsyncDocumentQuery(GetIndexName(isFullTextSearchEnabled)) + .OpenSubclause() + .Search(x => x.Query, searchParam) + .OrElse().WhereEquals(x => x.MessageId, searchParam) + .OrElse().WhereEquals(x => x.ConversationId, searchParam) + .CloseSubclause() + .ToQueryable(); + public async Task>> QueryMessagesByReceivingEndpoint(bool includeSystemMessages, string endpointName, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange timeSentRange, CancellationToken cancellationToken = default) { using var session = await sessionProvider.OpenSession(cancellationToken: cancellationToken);