diff --git a/.claude/rules/test-scenarios.md b/.claude/rules/test-scenarios.md new file mode 100644 index 00000000..8db3d456 --- /dev/null +++ b/.claude/rules/test-scenarios.md @@ -0,0 +1,6 @@ +--- +paths: + - "tests/**/*.fs" +--- + +@../../.github/instructions/test-scenarios.instructions.md diff --git a/.github/instructions/test-scenarios.instructions.md b/.github/instructions/test-scenarios.instructions.md new file mode 100644 index 00000000..c19cdc1f --- /dev/null +++ b/.github/instructions/test-scenarios.instructions.md @@ -0,0 +1,10 @@ +--- +applyTo: "tests/**/*.fs" +--- + +# Test scenarios + +* If all tests of an emulator test class read the same data, do not write the data in each test. Use a scenario: a `DatabaseTestApplicationFactory` descendant in `tests//Scenarios/`. +* If a test examines a write (create, replace, upsert, patch, or delete), write the data of the test in the test. +* Refer to the `TestApplicationFactory` of a scenario through its namespace, for example `OperationTestBase`. Open the namespace for the other types of the scenario. +* To write or change a scenario, use the `test-data-seeding` skill. diff --git a/.github/skills/test-data-seeding/SKILL.md b/.github/skills/test-data-seeding/SKILL.md new file mode 100644 index 00000000..64afb28e --- /dev/null +++ b/.github/skills/test-data-seeding/SKILL.md @@ -0,0 +1,186 @@ +--- +name: test-data-seeding +description: >- + Write a scenario for the emulator tests of this repository: a [] TestData record and a TestApplicationFactory + that creates the data in its constructor and writes the data in SeedDataAsync, and a test class that reads the data + with a record pattern. + Use this skill to write a test scenario, to add data to a scenario, or to make tests read seeded data. + Triggers: "seed test data", "create a test scenario", "TestData", "TestApplicationFactory", "scenario", + "integration test setup", "the same data in each test". +argument-hint: "Name of the scenario, the documents to write, the container" +--- + +# Test data seeding + +## Purpose + +A scenario writes the data that all tests of a class read. A scenario has two parts: + +1. A `[]` record with the name `TestData`. It holds the values that the tests read. +2. A class with the name `TestApplicationFactory`. It creates all data in its constructor and writes the data in `SeedDataAsync`. + +The tests read the data from `TestData`. They do not write the data. Thus each test starts with the same known data, and the body of a test shows only what the test examines. + +## When to use + +* All tests of an emulator test class read the same documents. +* The tests of a class read documents that a scenario writes already. Use that scenario, or add the documents to it. + +Do not use a scenario for a test that examines a write (create, replace, upsert, patch, or delete). This type of test writes its own data, for example with `OperationTestBase.SeedItemsAsync`. + +## Operating mode + +* Write the scenario directly. Do not ask for a confirmation. +* If a scenario writes the necessary documents already, use it. If it writes almost all of them, add the documents to it. Do not write a second scenario with the same data. +* Create all values in the constructor of the factory. `SeedDataAsync` only writes the values that the constructor created. +* Give fixed values to the documents. If the test project has Hedgehog generators, you can sample them with a fixed seed in the constructor. + +## File and namespace + +The file of a scenario is `tests//Scenarios/Scenario.fs`. + +* Declare the namespace `FSharp.Azure.Cosmos.Tests.Integration.Scenario`. +* Give the types the names `TestData` and `TestApplicationFactory`. The namespaces keep the types of the scenarios separate. +* Add the file to the `.fsproj` file before the files of the tests that use the scenario: + + ```xml + + + ``` + +The scenarios of `tests/Cosmos.Tests/Scenarios/` are examples: `SingleItemScenario.fs` and `MultipleItemsScenario.fs`. + +## TestData record + +* Make the record a `[]` record. +* Put into it only the values that the tests read. If no test reads a value of a document, keep the document in a private `let` binding of the factory. +* Do not put a value into `TestData` if the scenario does not write it. A test can use a value that is not in the database, for example the identifier of a missing item. Give this value a name that tells this fact: `MissingId`. +* Write a `///` comment for each field. + +## TestApplicationFactory + +```fsharp +namespace FSharp.Azure.Cosmos.Tests.Integration.ThreeItemsScenario + +open System.Collections.Immutable +open System.Threading +open System.Threading.Tasks + +open Microsoft.VisualStudio.TestTools.UnitTesting + +open FSharp.Azure.Cosmos +open FSharp.Azure.Cosmos.Tests +open FSharp.Azure.Cosmos.Tests.Integration + +/// The values that the tests of this scenario read. +[] +type TestData = { + /// The partition of the items that the scenario writes. + Partition : string + /// The items that the scenario writes, in the sequence of their creation. + Items : ImmutableArray +} + +/// +/// Writes three items into one partition of the container of . +/// +type TestApplicationFactory (testContext : TestContext) = + inherit DatabaseTestApplicationFactory (testContext) + + // The container of OperationTestBase + let containerId = "operation-tests" + + let partition = "integration" + + let items = + seq { 1..3 } + |> Seq.map (fun index -> { + id = $"item-{index}" + partitionKey = partition + name = $"item-{index}" + quantity = index + }) + |> ImmutableArray.CreateRange + + /// The values that the tests read. + member val TestData : TestData = { Partition = partition; Items = items } + + override this.SeedDataAsync (cancellationToken : CancellationToken) : Task = task { + let! container = this.GetOrCreateContainerAsync (containerId, "/partitionKey", cancellationToken) + + for seededItem in items do + let! response = + container.ExecuteAsync ( + create { + item seededItem + partitionKey seededItem.partitionKey + }, + cancellationToken + ) + + CosmosAssert.IsOk (response.Result, $"Seed create should succeed for '{seededItem.id}'.") + } +``` + +* Make the factory inherit from `DatabaseTestApplicationFactory`. +* Use `this.GetOrCreateContainerAsync` for each container. The container of `OperationTestBase` is `operation-tests`, with the partition key path `/partitionKey`. +* If the scenario creates more than one container, override `ContainerCount`. The fixture takes one partition of the emulator for each container. +* Write each item with its partition key. +* Examine the response of each write with `CosmosAssert.IsOk`. Thus a failed seed stops the test with a clear message. +* Give the cancellation token of `SeedDataAsync` to each call. +* If a client with other serializer options must write the documents, create the client in `SeedDataAsync` with `use`. Do not keep it in the factory. +* For a static collection of values, pipe a sequence expression into `ImmutableArray.CreateRange`. +* Each test has a database of its own. Thus an identifier must be unique only in the scenario. + +## Test class + +```fsharp +namespace FSharp.Azure.Cosmos.Tests.Integration + +open System +open System.Threading.Tasks + +open Microsoft.Azure.Cosmos +open Microsoft.VisualStudio.TestTools.UnitTesting + +open FSharp.Azure.Cosmos.Tests +open FSharp.Azure.Cosmos.Tests.Integration.ThreeItemsScenario + +[] +type ThreeItemsTests () = + inherit OperationTestBase () + + override _.CreateApplication context = ThreeItemsScenario.TestApplicationFactory (context) + + [] + member this.``A query of the partition returns the seeded items`` () : Task = task { + let { Partition = partition; Items = items } = this.Application.TestData + let! container = this.GetContainer () + + let options = QueryRequestOptions (PartitionKey = Nullable (PartitionKey partition)) + use iterator = container.GetItemQueryIterator("SELECT * FROM c", requestOptions = options) + let ids = ResizeArray() + + while iterator.HasMoreResults do + let! page = iterator.ReadNextAsync this.CancellationToken + ids.AddRange (page |> Seq.map _.id) + + CollectionAssert.AreEquivalent ( + items |> Seq.map _.id |> Seq.toArray, + ids.ToArray (), + "The query should return each seeded item one time." + ) + } +``` + +* Make the test class inherit from `OperationTestBase`. If the tests do not use the container of the operation tests, make the test class inherit from `IntegrationTestBase`. +* Override `CreateApplication`, and return a new `TestApplicationFactory`. +* Refer to `TestApplicationFactory` through the namespace of the scenario: `ThreeItemsScenario.TestApplicationFactory`. Thus the test class shows which scenario it uses. +* Open the namespace of the scenario for its other types. Thus the record pattern finds the labels of `TestData`. +* Read the values with a record pattern. Read only the fields that the test uses. Do not write `let data = this.Application.TestData`. +* Do not create the documents of the scenario in a test. A test can create its own input values, for example the parameters of a query. +* Give the class its test categories (refer to `test-categories.instructions.md`). + +## Lifetime + +`IntegrationTestBase` creates a new factory for each test in its `[]` method. The factory creates the database of the test, and `SeedDataAsync` writes the data. The `[]` method deletes the database. Thus a test cannot change the data of another test. diff --git a/tests/Cosmos.Tests/FSharp.Azure.Cosmos.Tests.fsproj b/tests/Cosmos.Tests/FSharp.Azure.Cosmos.Tests.fsproj index 37cbaf0c..3ce71123 100644 --- a/tests/Cosmos.Tests/FSharp.Azure.Cosmos.Tests.fsproj +++ b/tests/Cosmos.Tests/FSharp.Azure.Cosmos.Tests.fsproj @@ -29,6 +29,8 @@ + + diff --git a/tests/Cosmos.Tests/IterationExtensionsTests.fs b/tests/Cosmos.Tests/IterationExtensionsTests.fs index eac66657..798121cd 100644 --- a/tests/Cosmos.Tests/IterationExtensionsTests.fs +++ b/tests/Cosmos.Tests/IterationExtensionsTests.fs @@ -3,23 +3,21 @@ namespace FSharp.Azure.Cosmos.Tests.Integration open System.Threading.Tasks open FSharp.Control open FSharp.Azure.Cosmos.Tests +open FSharp.Azure.Cosmos.Tests.Integration.MultipleItemsScenario open Microsoft.Azure.Cosmos open Microsoft.Azure.Cosmos.Linq open Microsoft.VisualStudio.TestTools.UnitTesting [] type IterationExtensionsIntegrationTests () = - inherit OperationTestBase () + inherit OperationTestBase () - override _.CreateApplication context = MultipleItemsScenario (context) + override _.CreateApplication context = MultipleItemsScenario.TestApplicationFactory (context) [] member this.``FeedIterator AsAsyncEnumerable iterates seeded items`` () : Task = task { let! container = this.GetContainer () - let firstItem, secondItem = - match this.Application.SeededItems with - | [ firstItem; secondItem ] -> firstItem, secondItem - | seededItems -> failwith $"Expected exactly two seeded items but got {seededItems.Length}." + let { FirstSeededItem = firstItem; SecondSeededItem = secondItem } = this.Application.TestData let query = QueryDefinition("SELECT * FROM c WHERE c.partitionKey = @partitionKey").WithParameter("@partitionKey", "integration") @@ -41,10 +39,7 @@ type IterationExtensionsIntegrationTests () = [] member this.``IQueryable AsAsyncEnumerable iterates seeded items`` () : Task = task { let! container = this.GetContainer () - let firstItem, secondItem = - match this.Application.SeededItems with - | [ firstItem; secondItem ] -> firstItem, secondItem - | seededItems -> failwith $"Expected exactly two seeded items but got {seededItems.Length}." + let { FirstSeededItem = firstItem; SecondSeededItem = secondItem } = this.Application.TestData let queryable = container.GetItemLinqQueryable( diff --git a/tests/Cosmos.Tests/ReadManyOperationTests.fs b/tests/Cosmos.Tests/ReadManyOperationTests.fs index a1f991ce..886ab84f 100644 --- a/tests/Cosmos.Tests/ReadManyOperationTests.fs +++ b/tests/Cosmos.Tests/ReadManyOperationTests.fs @@ -1,63 +1,23 @@ namespace FSharp.Azure.Cosmos.Tests.Integration open System.Net -open System.Threading open System.Threading.Tasks open FSharp.Azure.Cosmos open FSharp.Azure.Cosmos.Tests +open FSharp.Azure.Cosmos.Tests.Integration.MultipleItemsScenario open Microsoft.Azure.Cosmos open Microsoft.VisualStudio.TestTools.UnitTesting -type MultipleItemsScenario (testContext : TestContext) as this = - inherit DatabaseTestApplicationFactory (testContext) - - let containerId = "operation-tests" - - let firstSeededItem : TestItem = { - id = $"{testContext.TestName}-readmany-1" - partitionKey = "integration" - name = "item-readmany-1" - quantity = 1 - } - - let secondSeededItem : TestItem = { - id = $"{testContext.TestName}-readmany-2" - partitionKey = "integration" - name = "item-readmany-2" - quantity = 2 - } - - member _.SeededItems = [ firstSeededItem; secondSeededItem ] - - override _.SeedDataAsync (cancellationToken : CancellationToken) : Task = task { - let! container = this.GetOrCreateContainerAsync (containerId, "/partitionKey", cancellationToken) - - for seededItem in this.SeededItems do - let! createResponse = - container.ExecuteAsync ( - create { - item seededItem - partitionKey seededItem.partitionKey - }, - cancellationToken - ) - - CosmosAssert.IsOk (createResponse.Result, $"ReadMany scenario seed create should succeed for '{seededItem.id}'.") - } - [] type ReadManyOperationIntegrationTests () = - inherit OperationTestBase () + inherit OperationTestBase () - override _.CreateApplication context = MultipleItemsScenario (context) + override _.CreateApplication context = MultipleItemsScenario.TestApplicationFactory (context) [] member this.``ReadMany execute returns matching items`` () : Task = task { let! container = this.GetContainer () - let firstItem, secondItem = - match this.Application.SeededItems with - | [ firstItem; secondItem ] -> firstItem, secondItem - | seededItems -> failwith $"Expected exactly two seeded items but got {seededItems.Length}." + let { FirstSeededItem = firstItem; SecondSeededItem = secondItem } = this.Application.TestData let! readManyResponse = container.ExecuteAsync ( diff --git a/tests/Cosmos.Tests/ReadOperationTests.fs b/tests/Cosmos.Tests/ReadOperationTests.fs index f328419b..a07a1587 100644 --- a/tests/Cosmos.Tests/ReadOperationTests.fs +++ b/tests/Cosmos.Tests/ReadOperationTests.fs @@ -2,52 +2,23 @@ namespace FSharp.Azure.Cosmos.Tests.Integration open System open System.Net -open System.Threading open System.Threading.Tasks open FSharp.Azure.Cosmos open FSharp.Azure.Cosmos.Tests +open FSharp.Azure.Cosmos.Tests.Integration.SingleItemScenario open Microsoft.Azure.Cosmos open Microsoft.VisualStudio.TestTools.UnitTesting -type SingleItemScenario (testContext : TestContext) as this = - inherit DatabaseTestApplicationFactory (testContext) - - let containerId = "operation-tests" - - let seededItem : TestItem = { - id = $"{testContext.TestName}-read" - partitionKey = "integration" - name = "item-read" - quantity = 1 - } - - member _.SeededItem = seededItem - - override _.SeedDataAsync (cancellationToken : CancellationToken) : Task = task { - let! container = this.GetOrCreateContainerAsync (containerId, "/partitionKey", cancellationToken) - - let! createResponse = - container.ExecuteAsync ( - create { - item seededItem - partitionKey seededItem.partitionKey - }, - cancellationToken - ) - - CosmosAssert.IsOk (createResponse.Result, "Read scenario seed create should succeed.") - } - [] type ReadOperationIntegrationTests () = - inherit OperationTestBase () + inherit OperationTestBase () - override _.CreateApplication context = SingleItemScenario (context) + override _.CreateApplication context = SingleItemScenario.TestApplicationFactory (context) [] member this.``Read execute returns existing and not found states`` () : Task = task { let! container = this.GetContainer () - let testItem = this.Application.SeededItem + let { SeededItem = testItem } = this.Application.TestData let! foundResponse = container.ExecuteAsync ( @@ -79,7 +50,7 @@ type ReadOperationIntegrationTests () = [] member this.``Read execute returns NotModified when eTag matches current item`` () : Task = task { let! container = this.GetContainer () - let testItem = this.Application.SeededItem + let { SeededItem = testItem } = this.Application.TestData let! foundResponse = container.ExecuteAsync ( @@ -128,7 +99,7 @@ type ReadOperationIntegrationTests () = [] member this.``ExecuteAsyncOption returns Some for existing item and None for missing item`` () : Task = task { let! container = this.GetContainer () - let testItem = this.Application.SeededItem + let { SeededItem = testItem } = this.Application.TestData let! foundOption = container.ExecuteAsyncOption ( @@ -158,7 +129,7 @@ type ReadOperationIntegrationTests () = [] member this.``ExecuteAsyncValueOption returns ValueSome for existing item and ValueNone for missing item`` () : Task = task { let! container = this.GetContainer () - let testItem = this.Application.SeededItem + let { SeededItem = testItem } = this.Application.TestData let! foundValueOption = container.ExecuteAsyncValueOption ( @@ -191,7 +162,7 @@ type ReadOperationIntegrationTests () = [] member this.``FeedIterator FirstAsync returns Ok for matching query and NotFound for empty query`` () : Task = task { let! container = this.GetContainer () - let testItem = this.Application.SeededItem + let { SeededItem = testItem } = this.Application.TestData let matchingQuery = QueryDefinition("SELECT * FROM c WHERE c.id = @id").WithParameter("@id", testItem.id) diff --git a/tests/Cosmos.Tests/Scenarios/MultipleItemsScenario.fs b/tests/Cosmos.Tests/Scenarios/MultipleItemsScenario.fs new file mode 100644 index 00000000..e0c24106 --- /dev/null +++ b/tests/Cosmos.Tests/Scenarios/MultipleItemsScenario.fs @@ -0,0 +1,61 @@ +namespace FSharp.Azure.Cosmos.Tests.Integration.MultipleItemsScenario + +open System.Threading +open System.Threading.Tasks + +open Microsoft.VisualStudio.TestTools.UnitTesting + +open FSharp.Azure.Cosmos +open FSharp.Azure.Cosmos.Tests +open FSharp.Azure.Cosmos.Tests.Integration + +/// The values that the tests of this scenario read. +[] +type TestData = { + /// The first item that the scenario writes. + FirstSeededItem : TestItem + /// The second item that the scenario writes, in the same partition as the first item. + SecondSeededItem : TestItem +} + +/// +/// Writes two items into one partition of the container of . +/// +type TestApplicationFactory (testContext : TestContext) = + inherit DatabaseTestApplicationFactory (testContext) + + // The container of OperationTestBase + let containerId = "operation-tests" + + let firstSeededItem : TestItem = { + id = $"{testContext.TestName}-readmany-1" + partitionKey = "integration" + name = "item-readmany-1" + quantity = 1 + } + + let secondSeededItem : TestItem = { + id = $"{testContext.TestName}-readmany-2" + partitionKey = "integration" + name = "item-readmany-2" + quantity = 2 + } + + /// The values that the tests read. + member val TestData : TestData = { FirstSeededItem = firstSeededItem; SecondSeededItem = secondSeededItem } + + override this.SeedDataAsync (cancellationToken : CancellationToken) : Task = task { + let! container = this.GetOrCreateContainerAsync (containerId, "/partitionKey", cancellationToken) + + for seededItem in [| firstSeededItem; secondSeededItem |] do + let! createResponse = + container.ExecuteAsync ( + create { + item seededItem + partitionKey seededItem.partitionKey + }, + cancellationToken + ) + + CosmosAssert.IsOk (createResponse.Result, $"ReadMany scenario seed create should succeed for '{seededItem.id}'.") + } diff --git a/tests/Cosmos.Tests/Scenarios/SingleItemScenario.fs b/tests/Cosmos.Tests/Scenarios/SingleItemScenario.fs new file mode 100644 index 00000000..4d241809 --- /dev/null +++ b/tests/Cosmos.Tests/Scenarios/SingleItemScenario.fs @@ -0,0 +1,51 @@ +namespace FSharp.Azure.Cosmos.Tests.Integration.SingleItemScenario + +open System.Threading +open System.Threading.Tasks + +open Microsoft.VisualStudio.TestTools.UnitTesting + +open FSharp.Azure.Cosmos +open FSharp.Azure.Cosmos.Tests +open FSharp.Azure.Cosmos.Tests.Integration + +/// The values that the tests of this scenario read. +[] +type TestData = { + /// The item that the scenario writes. + SeededItem : TestItem +} + +/// +/// Writes one item into the container of . +/// +type TestApplicationFactory (testContext : TestContext) = + inherit DatabaseTestApplicationFactory (testContext) + + // The container of OperationTestBase + let containerId = "operation-tests" + + let seededItem : TestItem = { + id = $"{testContext.TestName}-read" + partitionKey = "integration" + name = "item-read" + quantity = 1 + } + + /// The values that the tests read. + member val TestData : TestData = { SeededItem = seededItem } + + override this.SeedDataAsync (cancellationToken : CancellationToken) : Task = task { + let! container = this.GetOrCreateContainerAsync (containerId, "/partitionKey", cancellationToken) + + let! createResponse = + container.ExecuteAsync ( + create { + item seededItem + partitionKey seededItem.partitionKey + }, + cancellationToken + ) + + CosmosAssert.IsOk (createResponse.Result, "Read scenario seed create should succeed.") + }