diff --git a/Directory.Packages.props b/Directory.Packages.props index fbbe9a1..bef5653 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -12,7 +12,7 @@ - + diff --git a/MEVD/src/Chroma/Chroma.csproj b/MEVD/src/Chroma/Chroma.csproj index ae75898..0c488ad 100644 --- a/MEVD/src/Chroma/Chroma.csproj +++ b/MEVD/src/Chroma/Chroma.csproj @@ -1,7 +1,7 @@  - 1.0.2-preview + 1.1.0-preview CommunityToolkit.VectorData.Chroma $(AssemblyName) net10.0;net8.0;netstandard2.0;net462 diff --git a/MEVD/src/Chroma/ChromaCollection.cs b/MEVD/src/Chroma/ChromaCollection.cs index e59c563..b6bfc33 100644 --- a/MEVD/src/Chroma/ChromaCollection.cs +++ b/MEVD/src/Chroma/ChromaCollection.cs @@ -55,9 +55,6 @@ public class ChromaCollection : VectorStoreCollectionThe properties to create a BM25 index for when the collection is created. private readonly List _bm25Properties; - /// Whether the collection offers hybrid search, computed the first time it is asked. - private bool? _offersHybridSearch; - /// Whether the collection was disposed: it releases its share of the client only once. private int _disposed; @@ -411,7 +408,7 @@ public async IAsyncEnumerable> HybridSearchAsync { var index = await _records.FindBm25IndexAsync(textProperty.StorageName, cancellationToken).ConfigureAwait(false) - ?? throw new InvalidOperationException(NoBm25IndexMessage(textProperty)); + ?? throw new InvalidOperationException(await NoBm25IndexMessageAsync(textProperty, cancellationToken).ConfigureAwait(false)); return await _records.SearchAsync( new ChromaSearch { @@ -493,19 +490,29 @@ public override async IAsyncEnumerable GetAsync(Expression _offersHybridSearch ??= _chromaClient.Client.Options.IsChromaCloud && _bm25Properties.Count > 0; - - // Why the collection has no BM25 index on the property: on Chroma Cloud, the key of the index can be beyond its quota, or - // the collection was created without the index. - private string NoBm25IndexMessage(DataPropertyModel textProperty) + // Hybrid search needs the BM25 indexes, which distributed Chroma, the one Chroma Cloud runs on, and the servers that declare + // sparse vector indexes have. The client tells it from the server, the same whatever was called before, and keeps the answer: + // it is not kept here, because while the server does not answer, the client answers from its options and asks the server again + // later. An existing collection can have the indexes under keys that the client did not generate, so a missing index is + // reported by the hybrid search itself, not here. A method, for the callers that are not async: until the client has the answer + // of the server, it sends a request and waits for it, 5 seconds at most. + private bool OffersHybridSearch() + => _bm25Properties.Count > 0 && _chromaClient.Client.SupportsSparseVectorIndexes(); + + // Why the collection has no BM25 index on the property: the server has no sparse vector indexes, on Chroma Cloud the key of + // the index can be beyond its quota, or the collection was created without the index. + private async Task NoBm25IndexMessageAsync(DataPropertyModel textProperty, CancellationToken cancellationToken) { var message = $"The Chroma collection '{Name}' has no BM25 index on the text of the property '{textProperty.ModelName}', which hybrid search needs. "; + if (!await _chromaClient.Client.SupportsSparseVectorIndexesAsync(cancellationToken).ConfigureAwait(false)) + { + return message + "Only distributed Chroma, the one Chroma Cloud runs on, and the servers that declare sparse vector indexes have BM25 indexes. " + + "For distributed Chroma installed elsewhere, set Distributed=true in the connection string, or WithDistributedChroma() in the options of the client."; + } + if (!_chromaClient.Client.Options.IsChromaCloud) { - return message + "Only Chroma Cloud has BM25 indexes."; + return message + "The collection was created without it, for example by another client."; } var key = ChromaCollectionCreateMapping.GetBm25IndexKey(textProperty, _mapper.DocumentProperty); @@ -523,7 +530,7 @@ private string NoBm25IndexMessage(DataPropertyModel textProperty) serviceKey is not null ? null : serviceType == typeof(VectorStoreCollectionMetadata) ? _collectionMetadata : serviceType == typeof(ChromaClient) ? _chromaClient.Client : - serviceType == typeof(IKeywordHybridSearchable) ? (OffersHybridSearch ? this : null) : + serviceType == typeof(IKeywordHybridSearchable) ? (OffersHybridSearch() ? this : null) : serviceType.IsInstanceOfType(this) ? this : null; } diff --git a/MEVD/src/Chroma/README.md b/MEVD/src/Chroma/README.md index a0a8f6b..1d471c5 100644 --- a/MEVD/src/Chroma/README.md +++ b/MEVD/src/Chroma/README.md @@ -24,6 +24,7 @@ For more information, see the [Microsoft.Extensions.VectorData documentation](ht ## Limitations +- Hybrid search needs BM25 indexes, which distributed Chroma, the one Chroma Cloud runs on, and the servers that declare sparse vector indexes have. For distributed Chroma installed elsewhere, like with the Helm chart of Chroma or an operator, set `Distributed=true` in the connection string, or `WithDistributedChroma()` in the options of the client. On a single Chroma server, a collection does not offer `IKeywordHybridSearchable`, and a hybrid search throws an `InvalidOperationException` that says why. - On Chroma Cloud, a vector search returns at most 300 results, `Skip` included, the default quota of Chroma Cloud: beyond that, Chroma Cloud answers with a quota error. - Chroma does not store empty lists, so an empty array or list comes back as `null`. - A data property whose storage name is empty, or starts with `#`, `$` or `chroma:`, throws a `NotSupportedException` when the collection object is created. Chroma rejects the first ones in a write, and drops the `chroma:` keys from every read, so such a property would always come back as `null`. @@ -35,7 +36,7 @@ For more information, see the [Microsoft.Extensions.VectorData documentation](ht - On Chroma Cloud, a record has at most 32 metadata keys, and a null value counts as one. The provider writes every data property, null or not. A write with null values that would take a record beyond 32 keys throws an `ArgumentException`. With `WithReadsBeyondKeyQuota()`, the client reads such a record first and sends only the null values of the keys it has. Two writes of the same record at the same time can then mix. - On Chroma Cloud, a metadata key has at most 36 bytes in UTF-8. With a data property whose storage name is longer, the collection is created, but every upsert throws a `VectorStoreException` with the quota error of Chroma Cloud. - With two or more full-text properties, each has a BM25 index of its own, whose key is its storage name with `_bm25`. On Chroma Cloud, a storage name of more than 31 bytes makes that key longer than 36 bytes: the collection is created without that index, and a hybrid search on that property throws an `InvalidOperationException` that says why. -- On Chroma Cloud, `Contains` on the full-text property throws an `ArgumentException` for a text of fewer than 3 characters. Once Chroma Cloud has indexed the records, such a filter would find no record, and its negation every record. +- On distributed Chroma, the one Chroma Cloud runs on, `Contains` on the full-text property throws an `ArgumentException` for a text of fewer than 3 characters: on Chroma Cloud, and elsewhere with `Distributed=true` in the connection string or `WithDistributedChroma()` in the options of the client. Once the server has indexed the records, such a filter would find no record, and its negation every record. The connection string does not set `WithUpsertStrategy` and `WithReadsBeyondKeyQuota()`. Set them on the options of a `ChromaClient` created in code, and pass it to the vector store: diff --git a/MEVD/test/Chroma.ConformanceTests/ChromaHybridSearchTests.cs b/MEVD/test/Chroma.ConformanceTests/ChromaHybridSearchTests.cs index eee271e..0cfbdf8 100644 --- a/MEVD/test/Chroma.ConformanceTests/ChromaHybridSearchTests.cs +++ b/MEVD/test/Chroma.ConformanceTests/ChromaHybridSearchTests.cs @@ -9,55 +9,55 @@ namespace Chroma.ConformanceTests; /// -/// Hybrid search needs the Search API and the sparse vector indexes of Chroma, which only Chroma Cloud has: -/// these tests run only when Chroma:ConnectionString has a Chroma Cloud address, like https://api.trychroma.com, and are skipped otherwise. See the README for the settings. +/// Hybrid search needs the Search API and the sparse vector indexes of Chroma, which distributed Chroma, the one Chroma Cloud runs on, and the +/// servers that declare sparse vector indexes have: these tests run only against such a server, and are skipped otherwise. See the README for the settings. /// public class ChromaHybridSearchTests(ChromaHybridSearchTests.VectorAndStringFixture vectorAndStringFixture, ChromaHybridSearchTests.MultiTextFixture multiTextFixture) : HybridSearchTests(vectorAndStringFixture, multiTextFixture), IClassFixture, IClassFixture { - private const string SkipReason = "Hybrid search needs Chroma Cloud: set Chroma:ConnectionString to a Chroma Cloud address, like https://api.trychroma.com to run it."; + private const string SkipReason = "Hybrid search needs sparse vector indexes: set Chroma:ConnectionString to distributed Chroma, the one Chroma Cloud runs on, like https://api.trychroma.com or an address with Distributed=true, or to a server that declares sparse vector indexes, to run it."; public override Task HybridSearchAsync() { - Assert.SkipUnless(ChromaTestStore.IsChromaCloud, SkipReason); + Assert.SkipUnless(ChromaTestStore.Instance.SupportsSparseVectorIndexes, SkipReason); return base.HybridSearchAsync(); } public override Task HybridSearchAsync_with_filter() { - Assert.SkipUnless(ChromaTestStore.IsChromaCloud, SkipReason); + Assert.SkipUnless(ChromaTestStore.Instance.SupportsSparseVectorIndexes, SkipReason); return base.HybridSearchAsync_with_filter(); } public override Task HybridSearchAsync_with_top() { - Assert.SkipUnless(ChromaTestStore.IsChromaCloud, SkipReason); + Assert.SkipUnless(ChromaTestStore.Instance.SupportsSparseVectorIndexes, SkipReason); return base.HybridSearchAsync_with_top(); } public override Task HybridSearchAsync_with_Skip() { - Assert.SkipUnless(ChromaTestStore.IsChromaCloud, SkipReason); + Assert.SkipUnless(ChromaTestStore.Instance.SupportsSparseVectorIndexes, SkipReason); return base.HybridSearchAsync_with_Skip(); } public override Task HybridSearchAsync_with_multiple_keywords_ranks_matched_keywords_higher() { - Assert.SkipUnless(ChromaTestStore.IsChromaCloud, SkipReason); + Assert.SkipUnless(ChromaTestStore.Instance.SupportsSparseVectorIndexes, SkipReason); return base.HybridSearchAsync_with_multiple_keywords_ranks_matched_keywords_higher(); } public override Task HybridSearchAsync_with_multiple_text_properties() { - Assert.SkipUnless(ChromaTestStore.IsChromaCloud, SkipReason); + Assert.SkipUnless(ChromaTestStore.Instance.SupportsSparseVectorIndexes, SkipReason); return base.HybridSearchAsync_with_multiple_text_properties(); } public override Task HybridSearchAsync_without_explicitly_specified_property_fails() { - Assert.SkipUnless(ChromaTestStore.IsChromaCloud, SkipReason); + Assert.SkipUnless(ChromaTestStore.Instance.SupportsSparseVectorIndexes, SkipReason); return base.HybridSearchAsync_without_explicitly_specified_property_fails(); } diff --git a/MEVD/test/Chroma.ConformanceTests/README.md b/MEVD/test/Chroma.ConformanceTests/README.md index 2c5a6e2..e1a20d6 100644 --- a/MEVD/test/Chroma.ConformanceTests/README.md +++ b/MEVD/test/Chroma.ConformanceTests/README.md @@ -6,7 +6,7 @@ This project contains conformance tests for the Chroma Vector Store implementati By default, the tests will automatically use a testcontainer to spin up a Chroma instance. Docker must be running on your machine for this to work. -The hybrid search tests need Chroma Cloud, the only Chroma with the sparse vector indexes that hybrid search uses: they run only when `Chroma:ConnectionString` has a Chroma Cloud address, like `https://api.trychroma.com`, and are skipped otherwise. +The hybrid search tests need the sparse vector indexes that hybrid search uses, which distributed Chroma, the one Chroma Cloud runs on, and the servers that declare sparse vector indexes have: they run only when `Chroma:ConnectionString` points to such a server, like Chroma Cloud at `https://api.trychroma.com`, or distributed Chroma installed elsewhere with `Distributed=true`, and are skipped otherwise. ### Using an External Chroma Instance diff --git a/MEVD/test/Chroma.ConformanceTests/Support/ChromaTestStore.cs b/MEVD/test/Chroma.ConformanceTests/Support/ChromaTestStore.cs index cbbb30c..925012f 100644 --- a/MEVD/test/Chroma.ConformanceTests/Support/ChromaTestStore.cs +++ b/MEVD/test/Chroma.ConformanceTests/Support/ChromaTestStore.cs @@ -42,10 +42,10 @@ internal sealed class ChromaTestStore : TestStore public ChromaClient Client => this._client ?? throw new InvalidOperationException("Not initialized"); /// - /// Whether the tests run against Chroma Cloud, the only Chroma with the Search API and the sparse vector indexes that hybrid search needs. + /// Whether the server takes the sparse vector indexes that hybrid search needs: distributed Chroma, the one Chroma Cloud runs on, + /// and the servers that declare them. The client tells it when the store starts. /// - public static bool IsChromaCloud => ChromaTestEnvironment.IsConnectionStringDefined - && ChromaConfigurationOptions.FromConnectionString(ChromaTestEnvironment.ConnectionString!).Uri.Host.EndsWith(".trychroma.com", StringComparison.OrdinalIgnoreCase); + public bool SupportsSparseVectorIndexes { get; private set; } public ChromaVectorStore GetVectorStore(ChromaVectorStoreOptions options) => new(this.Client, ownsClient: false, options); // The client is shared, it's not owned by the vector store. @@ -69,6 +69,7 @@ protected override async Task StartAsync() } this._client = new ChromaClient(options); + this.SupportsSparseVectorIndexes = await this._client.SupportsSparseVectorIndexesAsync(); // It's a shared static instance, we don't want any of the tests to dispose the client. this.DefaultVectorStore = new ChromaVectorStore(this._client, ownsClient: false); diff --git a/MEVD/test/Chroma.ConformanceTests/testsettings.json b/MEVD/test/Chroma.ConformanceTests/testsettings.json index f3a1e9d..c69f0fe 100644 --- a/MEVD/test/Chroma.ConformanceTests/testsettings.json +++ b/MEVD/test/Chroma.ConformanceTests/testsettings.json @@ -1,6 +1,6 @@ { // Optional: Provide a connection string to use an external Chroma instance instead of testcontainers - // This is useful for running tests against Chroma Cloud, which hybrid search needs + // This is useful for running tests against distributed Chroma, the one Chroma Cloud runs on, which hybrid search needs // If not specified, tests will automatically use a testcontainer "Chroma": { "ConnectionString": "" diff --git a/MEVD/test/Chroma.UnitTests/ChromaCollectionTests.cs b/MEVD/test/Chroma.UnitTests/ChromaCollectionTests.cs index b905fac..6bc4a71 100644 --- a/MEVD/test/Chroma.UnitTests/ChromaCollectionTests.cs +++ b/MEVD/test/Chroma.UnitTests/ChromaCollectionTests.cs @@ -1,4 +1,4 @@ -// Licensed to the .NET Foundation under one or more agreements. +// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. using System; @@ -37,6 +37,7 @@ public ChromaCollectionTests() this._chromaClientMock .Setup(x => x.Options) .Returns(new ChromaConfigurationOptions("http://localhost:8000")); + this.SetupSparseVectorIndexes(false); this._chromaClientMock .Setup(x => x.GetCollectionClient(TestCollectionName)) .Returns(this._collectionClientMock.Object); @@ -263,30 +264,55 @@ public void GetServiceReturnsTheMetadataTheClientAndTheCollection() } [Theory] - [InlineData("https://api.trychroma.com", true)] - [InlineData("http://localhost:8000", false)] - public void GetServiceOffersHybridSearchOnlyWithFullTextPropertiesOnChromaCloud(string uri, bool offered) + [InlineData(true)] + [InlineData(false)] + public void GetServiceOffersHybridSearchOnlyWithFullTextPropertiesOnAServerWithSparseVectorIndexes(bool sparseVectorIndexes) { - // Arrange. - this._chromaClientMock - .Setup(x => x.Options) - .Returns(new ChromaConfigurationOptions(uri)); + // Arrange: the client tells whether the server takes sparse vector indexes, like the BM25 ones. + this.SetupSparseVectorIndexes(sparseVectorIndexes); using var fullText = this.CreateCollection(); using var noFullText = this.CreateCollection>(); // Act and assert. - Assert.Equal(offered, fullText.GetService(typeof(IKeywordHybridSearchable)) is not null); + Assert.Equal(sparseVectorIndexes, fullText.GetService(typeof(IKeywordHybridSearchable)) is not null); Assert.Null(noFullText.GetService(typeof(IKeywordHybridSearchable>))); } + [Fact] + public void GetServiceFollowsTheAnswerOfTheClient() + { + // Arrange: a server that did not answer yet, for which the client answers from its options, and answers again later. + this._chromaClientMock + .SetupSequence(x => x.SupportsSparseVectorIndexes()) + .Returns(false) + .Returns(true); + using var sut = this.CreateCollection(); + + // Act and assert: the collection keeps no answer of its own, the client keeps the answer of the server. + Assert.Null(sut.GetService(typeof(IKeywordHybridSearchable))); + Assert.NotNull(sut.GetService(typeof(IKeywordHybridSearchable))); + } + + [Fact] + public void GetServiceDoesNotAskTheClientWithoutFullTextProperties() + { + // Arrange. + this.SetupSparseVectorIndexes(true); + using var sut = this.CreateCollection>(); + + // Act. + Assert.Null(sut.GetService(typeof(IKeywordHybridSearchable>))); + + // Assert. + this._chromaClientMock.Verify(x => x.SupportsSparseVectorIndexes(), Times.Never); + } + [Fact] public void GetServiceOffersHybridSearchOnChromaCloudWhateverTheKeyOfTheBm25Index() { // Arrange: an existing collection can have its BM25 indexes under keys that the client did not generate, so a // generated key beyond 36 bytes does not hide hybrid search; the search itself reports a missing index. - this._chromaClientMock - .Setup(x => x.Options) - .Returns(new ChromaConfigurationOptions("https://api.trychroma.com")); + this.SetupChromaCloud(); using var oneFits = this.CreateCollection(); using var noneFits = this.CreateCollection(); @@ -477,22 +503,38 @@ public async Task HybridSearchThrowsWithoutABm25IndexAsync() // Act and assert. var exception = await Assert.ThrowsAsync(async () => await sut.HybridSearchAsync(new ReadOnlyMemory([1, 2, 3, 4]), ["pool"], top: 1, cancellationToken: this._testCancellationToken).ToListAsync()); - Assert.Contains("Only Chroma Cloud has BM25 indexes.", exception.Message); + Assert.Contains("Only distributed Chroma, the one Chroma Cloud runs on, and the servers that declare sparse vector indexes have BM25 indexes.", exception.Message); + Assert.Contains("Distributed=true", exception.Message); } [Fact] - public async Task HybridSearchOnChromaCloudSaysTheCollectionWasCreatedWithoutTheBm25IndexAsync() + public async Task HybridSearchOnDistributedChromaSaysTheCollectionWasCreatedWithoutTheBm25IndexAsync() { - // Arrange: Chroma Cloud, with a key that fits but no index, as in a collection that another client created. + // Arrange: distributed Chroma installed elsewhere, which has no quota on the keys, so even a long key is no reason. this._chromaClientMock .Setup(x => x.Options) - .Returns(new ChromaConfigurationOptions("https://api.trychroma.com")); + .Returns(new ChromaConfigurationOptions("http://localhost:8000").WithDistributedChroma()); + this.SetupSparseVectorIndexes(true); + using var sut = this.CreateHybridCollection(Bm25Index("Review_with_a_long_name_xxxxxxx_bm25", "Review_with_a_long_name_xxxxxxx")); + + // Act and assert. + var exception = await Assert.ThrowsAsync(async () => await sut.HybridSearchAsync(new ReadOnlyMemory([1, 2, 3, 4]), ["pool"], top: 1, new() { AdditionalProperty = h => h.Description }, this._testCancellationToken).ToListAsync()); + Assert.Contains("created without it", exception.Message); + Assert.DoesNotContain("bytes", exception.Message); + Assert.DoesNotContain("Only distributed Chroma", exception.Message); + } + + [Fact] + public async Task HybridSearchOnChromaCloudSaysTheCollectionWasCreatedWithoutTheBm25IndexAsync() + { + // Arrange: Chroma Cloud, with a key that fits but no index, as in a collection that another client created. + this.SetupChromaCloud(); using var sut = this.CreateHybridCollection(); // Act and assert. var exception = await Assert.ThrowsAsync(async () => await sut.HybridSearchAsync(new ReadOnlyMemory([1, 2, 3, 4]), ["pool"], top: 1, cancellationToken: this._testCancellationToken).ToListAsync()); Assert.Contains("created without it", exception.Message); - Assert.DoesNotContain("Only Chroma Cloud", exception.Message); + Assert.DoesNotContain("Only distributed Chroma", exception.Message); } [Fact] @@ -510,9 +552,7 @@ public async Task HybridSearchThrowsWithoutAnIndexOnTheChosenPropertyAsync() public async Task HybridSearchSaysWhyChromaCloudHasNoBm25IndexOnALongStorageNameAsync() { // Arrange: Chroma Cloud, with only the index of the property that fits. - this._chromaClientMock - .Setup(x => x.Options) - .Returns(new ChromaConfigurationOptions("https://api.trychroma.com")); + this.SetupChromaCloud(); using var sut = this.CreateHybridCollection(Bm25Index("Review_with_a_long_name_xxxxxxx_bm25", "Review_with_a_long_name_xxxxxxx")); // Act and assert. @@ -681,6 +721,26 @@ private Func SetupQuery(params ChromaCollectionQueryEntry[] results return () => query!; } + // Whether the server takes sparse vector indexes, as the client tells it. + private void SetupSparseVectorIndexes(bool sparseVectorIndexes) + { + this._chromaClientMock + .Setup(x => x.SupportsSparseVectorIndexes()) + .Returns(sparseVectorIndexes); + this._chromaClientMock + .Setup(x => x.SupportsSparseVectorIndexesAsync(this._testCancellationToken)) + .ReturnsAsync(sparseVectorIndexes); + } + + // Chroma Cloud, which takes sparse vector indexes. + private void SetupChromaCloud() + { + this._chromaClientMock + .Setup(x => x.Options) + .Returns(new ChromaConfigurationOptions("https://api.trychroma.com")); + this.SetupSparseVectorIndexes(true); + } + // The collection is not read. private ChromaCollection CreateCollection() where TKey : notnull