diff --git a/README.md b/README.md index adfd9206..27a74943 100644 --- a/README.md +++ b/README.md @@ -446,6 +446,48 @@ session.toolExecutionDelegate = ToolExecutionObserver() > Tool execution delegates are an AnyLanguageModel extension. > See [Differences from Foundation Models](#differences-from-foundation-models). +### Dynamic Instructions + +`DynamicInstructions` lets a session change the instructions and tools +for each request to the model, +without creating a new session: + +```swift +final class CurrentAppState { + var canCheckWeather = false +} + +struct CurrentAppInstructions: DynamicInstructions { + let state: CurrentAppState + + var body: some DynamicInstructions { + Instructions("Help with the currently visible app.") + if state.canCheckWeather { + WeatherTool() + } + } +} + +let state = CurrentAppState() +let session = LanguageModelSession( + model: model, + dynamicInstructions: CurrentAppInstructions(state: state), + history: savedHistory +) +``` + +The session evaluates the body before every request to the model, +including the request that continues a response after tool calls. +The resolved instructions are sent with each request +but never become part of the session's transcript. + +> [!NOTE] +> Dynamic instructions follow the Foundation Models 27 API. +> `SystemLanguageModel` supports them only in apps built with Swift 6.4 or later +> that run on OS 27 or later, and not on tvOS. +> Otherwise, it throws `SystemLanguageModel.Error.dynamicInstructionsUnavailable` +> for a session with dynamic instructions. + ### Reasoning in the transcript Reasoning is transcript content, separate from the answer in `response.content`. @@ -582,6 +624,9 @@ say which API they follow. - `Usage` and the `usage` properties: [token usage](#token-usage), which follows the Foundation Models 27 API. +- `DynamicInstructions`, its builder, and `LanguageModelSession.init(model:dynamicInstructions:history:)`: + [dynamic instructions](#dynamic-instructions), + which follow the Foundation Models 27 API. - `Transcript.Entry.reasoning` and `Transcript.Reasoning`: [reasoning in the transcript](#reasoning-in-the-transcript), which follows the Foundation Models 27 API. @@ -590,7 +635,8 @@ say which API they follow. for language models defined outside AnyLanguageModel. - `Codable` conformance for `GeneratedContent`, `GenerationID`, `Usage`, and the types nested in `Transcript`. -- `GeneratedContentError`, `Transcript.ReasoningReplayError`, and each provider's error type. +- `GeneratedContentError`, `Transcript.ReasoningReplayError`, `SystemLanguageModel.Error`, + and each provider's error type. - `JSONValue`: JSON values for provider options such as `extraBody`. - Smaller additions to existing types, diff --git a/Sources/AnyLanguageModel/LanguageModelSession.swift b/Sources/AnyLanguageModel/LanguageModelSession.swift index 7e2c2c41..4f08efc6 100644 --- a/Sources/AnyLanguageModel/LanguageModelSession.swift +++ b/Sources/AnyLanguageModel/LanguageModelSession.swift @@ -85,6 +85,9 @@ public final class LanguageModelSession: @unchecked Sendable { /// The tools that the model can call during the session. /// + /// For a session created with dynamic instructions, this is empty. + /// Call ``resolvedRequestContext()`` to get the tools for the next request. + /// /// - Note: This property is exclusive to AnyLanguageModel /// and using it means your code is no longer drop-in compatible /// with the Foundation Models framework. @@ -94,6 +97,9 @@ public final class LanguageModelSession: @unchecked Sendable { /// The instructions for the session, if any. /// + /// For a session created with dynamic instructions, this is `nil`. + /// Call ``resolvedRequestContext()`` to get the instructions for the next request. + /// /// - Note: This property is exclusive to AnyLanguageModel /// and using it means your code is no longer drop-in compatible /// with the Foundation Models framework. @@ -101,6 +107,14 @@ public final class LanguageModelSession: @unchecked Sendable { /// can read the session's instructions. public let instructions: Instructions? + private let dynamicInstructions: AnyDynamicInstructions? + private let dynamicInstructionsLock = NSLock() + + /// Whether the session resolves its instructions and tools before each request. + nonisolated var usesDynamicInstructions: Bool { + dynamicInstructions != nil + } + /// A delegate that observes and controls tool execution. /// /// Set this property to intercept tool calls, provide custom output, @@ -151,7 +165,29 @@ public final class LanguageModelSession: @unchecked Sendable { /// It's public so that language models outside this module /// can read the inputs for each request. nonisolated public func resolvedRequestContext() -> RequestContext { - RequestContext(transcript: transcript, instructions: instructions, tools: tools) + guard let dynamicInstructions else { + return RequestContext(transcript: transcript, instructions: instructions, tools: tools) + } + + // Evaluate the body for this request. + // The resolved instructions go into the request's transcript, + // never into the session's. + let resolved = dynamicInstructionsLock.withLock { + dynamicInstructions.resolveForRequest() + } + var requestTranscript = transcript + if let instructions = resolved.instructions { + let instructionsEntry = Transcript.Entry.instructions( + Transcript.Instructions( + segments: [.text(Transcript.TextSegment(content: instructions.description))], + toolDefinitions: resolved.tools + .filter(\.includesSchemaInInstructions) + .map { Transcript.ToolDefinition(tool: $0) } + ) + ) + requestTranscript = Transcript(entries: [instructionsEntry] + requestTranscript) + } + return RequestContext(transcript: requestTranscript, instructions: resolved.instructions, tools: resolved.tools) } /// Creates a session with a model, tools, @@ -213,14 +249,53 @@ public final class LanguageModelSession: @unchecked Sendable { self.init(model: model, tools: tools, instructions: nil, transcript: transcript) } + /// Creates a session whose instructions and tools are resolved before each model request. + /// + /// The session evaluates the body of `dynamicInstructions` + /// before every request to the model, + /// including the request that continues a response after tool calls. + /// The resolved instructions are sent with each request + /// but never become part of the session's transcript. + /// + /// - Parameters: + /// - model: The language model to use. + /// - dynamicInstructions: The instructions and tools to resolve for each request. + /// - history: Earlier transcript entries to continue from. + /// Instructions entries in the history are left out, + /// because the dynamic instructions take their place. + /// + /// - Note: This API is exclusive to AnyLanguageModel on OS 26. + /// It follows the Foundation Models 27 `LanguageModelSession.init(model:dynamicInstructions:history:)` API, + /// so code that uses it ports to Foundation Models on OS 27. + /// Unlike Foundation Models, AnyLanguageModel requires the `model` argument. + public convenience init( + model: any LanguageModel, + dynamicInstructions: sending some DynamicInstructions, + history: some Collection = [] + ) { + let history = history.filter { entry in + if case .instructions = entry { return false } + return true + } + self.init( + model: model, + tools: [], + instructions: nil, + transcript: Transcript(entries: history), + dynamicInstructions: AnyDynamicInstructions(dynamicInstructions) + ) + } + private init( model: any LanguageModel, tools: [any Tool], instructions: Instructions?, - transcript: Transcript + transcript: Transcript, + dynamicInstructions: AnyDynamicInstructions? = nil ) { self.model = model self.tools = tools + self.dynamicInstructions = dynamicInstructions let resolvedInstructions = instructions ?? Self.instructions(from: transcript) self.instructions = resolvedInstructions diff --git a/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift b/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift index 8def36c2..87763ed6 100644 --- a/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift @@ -296,6 +296,21 @@ for session: LanguageModelSession, prompt: Prompt ) throws -> FoundationModels.LanguageModelSession { + #if compiler(>=6.4) && !os(tvOS) + if #available(macOS 27.0, iOS 27.0, visionOS 27.0, watchOS 27.0, *) { + return makeFoundationModelsSession( + model: systemModel, + session: session, + prompt: prompt + ) + } + #endif + + // Before OS 27, Foundation Models can't resolve instructions and tools + // again for the request that continues after tool calls. + guard !session.usesDynamicInstructions else { + throw SystemLanguageModel.Error.dynamicInstructionsUnavailable + } let requestContext = session.resolvedRequestContext() return FoundationModels.LanguageModelSession( model: systemModel, @@ -331,13 +346,71 @@ return Transcript(entries: transcript.dropLast()) } + @available(macOS 26.0, iOS 26.0, tvOS 26.0, visionOS 26.0, *) + @available(watchOS, unavailable) + extension SystemLanguageModel { + /// An error from the system language model. + /// + /// - Note: This API is exclusive to AnyLanguageModel + /// and using it means your code is no longer drop-in compatible + /// with the Foundation Models framework. + /// Foundation Models 27 has a `SystemLanguageModel.Error` type with other cases. + public enum Error: LocalizedError, Sendable, Equatable { + /// The session uses dynamic instructions, + /// which the system language model supports only in apps built with Swift 6.4 or later + /// that run on OS 27 or later, and not on tvOS. + case dynamicInstructionsUnavailable + + public var errorDescription: String? { + switch self { + case .dynamicInstructionsUnavailable: + "Dynamic instructions require an app built with Swift 6.4 or later that runs Foundation Models on OS 27 or later, and aren't available on tvOS." + } + } + } + } + #if compiler(>=6.4) && !os(tvOS) + /// Dynamic instructions that resolve an AnyLanguageModel session + /// each time Foundation Models evaluates them. + @available(macOS 27.0, iOS 27.0, visionOS 27.0, watchOS 27.0, *) + struct FoundationModelsDynamicInstructionsAdapter: FoundationModels.DynamicInstructions { + let session: LanguageModelSession + + var body: some FoundationModels.DynamicInstructions { + let requestContext = session.resolvedRequestContext() + if let instructions = requestContext.instructions { + instructions.toFoundationModels() + } + requestContext.tools.toFoundationModels() + } + } + @available(macOS 27.0, iOS 27.0, visionOS 27.0, watchOS 27.0, *) func makeFoundationModelsSession( model: Model, session: LanguageModelSession, prompt: Prompt ) -> FoundationModels.LanguageModelSession { + if session.usesDynamicInstructions { + // Foundation Models evaluates the dynamic instructions itself, + // so the history leaves out the instructions entry. + let history = fmTranscriptDroppingDuplicatePrompt( + Transcript( + entries: session.transcript.filter { entry in + if case .instructions = entry { return false } + return true + } + ), + prompt: prompt + ).toFoundationModels(instructions: nil, toolDefinitions: []) + return FoundationModels.LanguageModelSession( + model: model, + dynamicInstructions: FoundationModelsDynamicInstructionsAdapter(session: session), + history: history + ) + } + let requestContext = session.resolvedRequestContext() return FoundationModels.LanguageModelSession( model: model, diff --git a/Tests/AnyLanguageModelTests/APICompatibilityAnyLanguageModelTests.swift b/Tests/AnyLanguageModelTests/APICompatibilityAnyLanguageModelTests.swift index 55106cdd..18bf1cba 100644 --- a/Tests/AnyLanguageModelTests/APICompatibilityAnyLanguageModelTests.swift +++ b/Tests/AnyLanguageModelTests/APICompatibilityAnyLanguageModelTests.swift @@ -10,6 +10,18 @@ import Testing return false }() + @available(macOS 27.0, iOS 27.0, visionOS 27.0, *) + private struct CompatibilityDynamicInstructions: DynamicInstructions { + let includeDetail: Bool + + var body: some DynamicInstructions { + Instructions("You are a helpful assistant.") + if includeDetail { + Instructions("Include useful detail.") + } + } + } + @available(macOS 26.0, iOS 26.0, tvOS 26.0, visionOS 26.0, *) @Test("AnyLanguageModel Drop-In Compatibility", .enabled(if: isSystemLanguageModelAvailable)) func anyLanguageModelCompatibility() async throws { @@ -19,6 +31,14 @@ import Testing instructions: Instructions("You are a helpful assistant.") ) + if #available(macOS 27.0, iOS 27.0, visionOS 27.0, *) { + _ = LanguageModelSession( + model: model, + dynamicInstructions: CompatibilityDynamicInstructions(includeDetail: true), + history: session.transcript + ) + } + let options = GenerationOptions(temperature: 0.7) let response = try await session.respond(options: options) { Prompt("Say 'Hello'") diff --git a/Tests/AnyLanguageModelTests/APICompatibilityFoundationModelsTests.swift b/Tests/AnyLanguageModelTests/APICompatibilityFoundationModelsTests.swift index 60943da5..8142648e 100644 --- a/Tests/AnyLanguageModelTests/APICompatibilityFoundationModelsTests.swift +++ b/Tests/AnyLanguageModelTests/APICompatibilityFoundationModelsTests.swift @@ -10,6 +10,22 @@ import Testing return false }() + #if compiler(>=6.4) + #if os(macOS) || os(iOS) || os(visionOS) + @available(macOS 27.0, iOS 27.0, visionOS 27.0, *) + private struct CompatibilityDynamicInstructions: DynamicInstructions { + let includeDetail: Bool + + var body: some DynamicInstructions { + Instructions("You are a helpful assistant.") + if includeDetail { + Instructions("Include useful detail.") + } + } + } + #endif + #endif + @available(macOS 26.0, iOS 26.0, tvOS 26.0, visionOS 26.0, *) @Test( "FoundationModels Drop-In Compatibility", @@ -22,6 +38,18 @@ import Testing instructions: Instructions("You are a helpful assistant.") ) + #if compiler(>=6.4) + #if os(macOS) || os(iOS) || os(visionOS) + if #available(macOS 27.0, iOS 27.0, visionOS 27.0, *) { + _ = LanguageModelSession( + model: model, + dynamicInstructions: CompatibilityDynamicInstructions(includeDetail: true), + history: session.transcript + ) + } + #endif + #endif + let options = GenerationOptions(temperature: 0.7) let response = try await session.respond(options: options) { Prompt("Say 'Hello'") diff --git a/Tests/AnyLanguageModelTests/DynamicInstructionsTests.swift b/Tests/AnyLanguageModelTests/DynamicInstructionsTests.swift new file mode 100644 index 00000000..af43ad37 --- /dev/null +++ b/Tests/AnyLanguageModelTests/DynamicInstructionsTests.swift @@ -0,0 +1,591 @@ +import Foundation +import Testing + +@testable import AnyLanguageModel + +@Suite("Dynamic instructions") +struct DynamicInstructionsTests { + @Test func dynamicSessionLeavesInstructionsOutOfHistoryAndSessionProperties() { + let state = DynamicFixtureState() + let history: [Transcript.Entry] = [ + .instructions(Transcript.Instructions(segments: [.text(.init(content: "Old"))], toolDefinitions: [])), + .prompt(Transcript.Prompt(segments: [.text(.init(content: "Hello"))])), + ] + let session = LanguageModelSession( + model: DynamicContextModel(state: state, continuesAfterTool: false), + dynamicInstructions: FixtureDynamicInstructions(state: state), + history: history + ) + + #expect(session.instructions == nil) + #expect(session.tools.isEmpty) + #expect(session.transcript.count == 1) + let context = session.resolvedRequestContext() + #expect(context.instructions?.description == "Instructions A") + #expect(context.transcript.count == 2) + } + + @Test func bodyReevaluatesForEveryNonstreamingRequest() async throws { + let state = DynamicFixtureState() + let model = DynamicContextModel(state: state, continuesAfterTool: false) + let session = LanguageModelSession( + model: model, + dynamicInstructions: FixtureDynamicInstructions(state: state) + ) + + #expect(state.evaluationCount == 0) + _ = try await session.respond(to: "First") + state.select(.b) + _ = try await session.respond(to: "Second") + + #expect( + model.snapshots.withLock { $0 } == [ + .init(instructions: "Instructions A", tools: ["tool-a"]), + .init(instructions: "Instructions B", tools: ["tool-b"]), + ] + ) + #expect(state.evaluationCount == 2) + } + + @Test func bodyReevaluatesForEveryStreamingRequest() async throws { + let state = DynamicFixtureState() + let model = DynamicContextModel(state: state, continuesAfterTool: false) + let session = LanguageModelSession( + model: model, + dynamicInstructions: FixtureDynamicInstructions(state: state) + ) + + #expect(state.evaluationCount == 0) + _ = try await session.streamResponse(to: "First").collect() + state.select(.b) + _ = try await session.streamResponse(to: "Second").collect() + + #expect( + model.snapshots.withLock { $0 } == [ + .init(instructions: "Instructions A", tools: ["tool-a"]), + .init(instructions: "Instructions B", tools: ["tool-b"]), + ] + ) + #expect(state.evaluationCount == 2) + } + + @Test(arguments: [false, true]) + func toolContinuationReevaluatesAndExecutesProducingSnapshot(streaming: Bool) async throws { + let state = DynamicFixtureState() + let model = DynamicContextModel(state: state, continuesAfterTool: true) + let session = LanguageModelSession( + model: model, + dynamicInstructions: FixtureDynamicInstructions(state: state) + ) + + if streaming { + _ = try await session.streamResponse(to: "Use a tool").collect() + } else { + _ = try await session.respond(to: "Use a tool") + } + + #expect( + model.snapshots.withLock { $0 } == [ + .init(instructions: "Instructions A", tools: ["tool-a"]), + .init(instructions: "Instructions B", tools: ["tool-b"]), + ] + ) + #expect(state.executedTools == ["tool-a"]) + #expect(state.evaluationCount == 2) + #expect(session.transcript.count == 4) + guard case .prompt = session.transcript[0], + case .toolCalls(let calls) = session.transcript[1], + case .toolOutput(let output) = session.transcript[2], + case .response = session.transcript[3] + else { + Issue.record("Expected prompt, tool call, tool output, and response") + return + } + #expect(calls.first?.id == output.id) + #expect(output.toolName == "tool-a") + } + + @Test func historyDoesNotPersistDynamicInstructionsAndRehydratesWithCurrentState() async throws { + let state = DynamicFixtureState() + let firstModel = DynamicContextModel(state: state, continuesAfterTool: false) + let firstSession = LanguageModelSession( + model: firstModel, + dynamicInstructions: FixtureDynamicInstructions(state: state) + ) + _ = try await firstSession.respond(to: "First") + + #expect( + !firstSession.transcript.contains { + if case .instructions = $0 { true } else { false } + } + ) + + state.select(.b) + let restoredModel = DynamicContextModel(state: state, continuesAfterTool: false) + let restoredSession = LanguageModelSession( + model: restoredModel, + dynamicInstructions: FixtureDynamicInstructions(state: state), + history: firstSession.transcript + ) + _ = try await restoredSession.respond(to: "Second") + + #expect( + restoredModel.snapshots.withLock { $0 } == [ + .init(instructions: "Instructions B", tools: ["tool-b"]) + ] + ) + #expect(restoredSession.transcript.count == 4) + #expect( + !restoredSession.transcript.contains { + if case .instructions = $0 { true } else { false } + } + ) + } + + @Test func builderComposesNestedConditionalEmptyAndToolArrayContent() { + let state = DynamicFixtureState() + let enabled = true + let dynamic = AnyDynamicInstructions(erasing: FixtureComposition(state: state, enabled: enabled)) + let session = LanguageModelSession( + model: DynamicContextModel(state: state, continuesAfterTool: false), + dynamicInstructions: dynamic + ) + + let context = session.resolvedRequestContext() + + #expect(context.instructions?.description == "Outer\nNested\nFor each") + #expect(context.tools.map(\.name) == ["tool-a", "tool-b"]) + #expect(session.transcript.isEmpty) + } + + @Test func staticSessionRequestContextPreservesExistingBehavior() { + let state = DynamicFixtureState() + let tool = FixtureTool(name: "static-tool", state: state) + let session = LanguageModelSession( + model: DynamicContextModel(state: state, continuesAfterTool: false), + tools: [tool], + instructions: "Static" + ) + + let context = session.resolvedRequestContext() + + #expect(context.instructions?.description == "Static") + #expect(context.tools.map(\.name) == ["static-tool"]) + #expect(context.transcript == session.transcript) + #expect(session.instructions?.description == "Static") + #expect(session.tools.map(\.name) == ["static-tool"]) + } + + @Test func failedResponseDoesNotReplayCompletedDynamicToolSideEffect() async throws { + let state = DynamicFixtureState() + let model = FailingAfterToolModel(state: state) + let session = LanguageModelSession( + model: model, + dynamicInstructions: FixtureDynamicInstructions(state: state) + ) + + await #expect(throws: DynamicFixtureError.failed) { + _ = try await session.respond(to: "Fail after tool") + } + _ = try await session.respond(to: "Retry") + + #expect(state.executedTools == ["tool-a"]) + #expect( + model.snapshots.withLock { $0 } == [ + .init(instructions: "Instructions A", tools: ["tool-a"]), + .init(instructions: "Instructions B", tools: ["tool-b"]), + ] + ) + } + + @Test func cancelledResponseDoesNotReplayCompletedDynamicToolSideEffect() async throws { + let state = DynamicFixtureState() + let control = DynamicCancellationControl() + let model = CancellingAfterToolModel(state: state, control: control) + let session = LanguageModelSession( + model: model, + dynamicInstructions: FixtureDynamicInstructions(state: state) + ) + var started = control.started.makeAsyncIterator() + + let response = Task { + try await session.respond(to: "Cancel after tool") + } + _ = await started.next() + response.cancel() + + await #expect(throws: CancellationError.self) { + _ = try await response.value + } + _ = try await session.respond(to: "Retry") + + #expect(state.executedTools == ["tool-a"]) + #expect( + model.snapshots.withLock { $0 } == [ + .init(instructions: "Instructions A", tools: ["tool-a"]), + .init(instructions: "Instructions B", tools: ["tool-b"]), + ] + ) + } +} + +private struct FixtureDynamicInstructions: DynamicInstructions { + let state: DynamicFixtureState + + var body: some DynamicInstructions { + let snapshot = state.snapshotForEvaluation() + Instructions(snapshot.instructions) + [snapshot.tool] + } +} + +private struct FixtureComposition: DynamicInstructions { + let state: DynamicFixtureState + let enabled: Bool + + var body: some DynamicInstructions { + Instructions("Outer") + if enabled { + NestedFixtureInstructions(state: state) + } + EmptyDynamicInstructions() + ForEach([FixtureInstruction(id: 1, text: "For each")]) { item in + Instructions(item.text) + } + } +} + +private struct FixtureInstruction: Identifiable { + let id: Int + let text: String +} + +private struct NestedFixtureInstructions: DynamicInstructions { + let state: DynamicFixtureState + + var body: some DynamicInstructions { + Instructions("Nested") + [ + FixtureTool(name: "tool-a", state: state), + FixtureTool(name: "tool-b", state: state), + ] as [any Tool] + } +} + +private final class DynamicFixtureState: @unchecked Sendable { + enum Selection: Sendable { + case a + case b + } + + struct Storage: Sendable { + var selection = Selection.a + var evaluationCount = 0 + var executedTools: [String] = [] + } + + struct Snapshot: Sendable { + let instructions: String + let tool: any Tool + } + + private let storage = Locked(Storage()) + + var evaluationCount: Int { + storage.withLock { $0.evaluationCount } + } + + var executedTools: [String] { + storage.withLock { $0.executedTools } + } + + func select(_ selection: Selection) { + storage.withLock { $0.selection = selection } + } + + func snapshotForEvaluation() -> Snapshot { + storage.withLock { storage in + storage.evaluationCount += 1 + switch storage.selection { + case .a: + return Snapshot( + instructions: "Instructions A", + tool: FixtureTool(name: "tool-a", state: self) + ) + case .b: + return Snapshot( + instructions: "Instructions B", + tool: FixtureTool(name: "tool-b", state: self) + ) + } + } + } + + func recordExecution(_ name: String) { + storage.withLock { $0.executedTools.append(name) } + } +} + +private struct FixtureTool: Tool { + let name: String + let description = "Records which request-scoped tool instance executed" + let state: DynamicFixtureState + + typealias Arguments = GeneratedContent + + var parameters: GenerationSchema { + GeneratedContent.generationSchema + } + + func call(arguments: GeneratedContent) async throws -> String { + state.recordExecution(name) + return name + } +} + +private struct DynamicRequestSnapshot: Sendable, Equatable { + let instructions: String? + let tools: [String] +} + +private struct DynamicContextModel: LanguageModel { + typealias UnavailableReason = Never + + let state: DynamicFixtureState + let continuesAfterTool: Bool + let snapshots = Locked<[DynamicRequestSnapshot]>([]) + + func respond( + within session: LanguageModelSession, + to prompt: Prompt, + generating type: Content.Type, + includeSchemaInPrompt: Bool, + options: GenerationOptions + ) async throws -> LanguageModelSession.Response where Content: Generable { + let result = try await run(session: session, type: type) + return .init( + content: result.content, + rawContent: result.raw, + transcriptEntries: result.entries + ) + } + + func streamResponse( + within session: LanguageModelSession, + to prompt: Prompt, + generating type: Content.Type, + includeSchemaInPrompt: Bool, + options: GenerationOptions + ) -> sending LanguageModelSession.ResponseStream where Content: Generable { + let stream = AsyncThrowingStream.Snapshot, any Error> { + continuation in + Task { + do { + let result = try await run(session: session, type: type) + continuation.yield( + .init( + content: result.content.asPartiallyGenerated(), + rawContent: result.raw, + transcriptEntries: ArraySlice(result.entries) + ) + ) + continuation.finish() + } catch { + continuation.finish(throwing: error) + } + } + } + return .init(stream: stream) + } + + private func run( + session: LanguageModelSession, + type: Content.Type + ) async throws -> (content: Content, raw: GeneratedContent, entries: ArraySlice) { + let first = session.resolvedRequestContext() + record(first) + var entries: [Transcript.Entry] = [] + + if continuesAfterTool { + let tool = try #require(first.tools.first) + state.select(.b) + let call = Transcript.ToolCall( + id: "request-a-call", + toolName: tool.name, + arguments: GeneratedContent(properties: [:]) + ) + let output = Transcript.ToolOutput( + id: call.id, + toolName: call.toolName, + segments: try await tool.makeOutputSegments(from: call.arguments) + ) + entries.append(.toolCalls(.init(id: "request-a-calls", [call]))) + entries.append(.toolOutput(output)) + + let continuation = session.resolvedRequestContext() + record(continuation) + } + + let raw = GeneratedContent("Done") + return (try Content(raw), raw, ArraySlice(entries)) + } + + private func record(_ context: LanguageModelSession.RequestContext) { + snapshots.withLock { + $0.append( + .init( + instructions: context.instructions?.description, + tools: context.tools.map(\.name) + ) + ) + } + } +} + +private enum DynamicFixtureError: Error, Equatable { + case failed +} + +private struct FailingAfterToolModel: LanguageModel { + typealias UnavailableReason = Never + + let state: DynamicFixtureState + let snapshots = Locked<[DynamicRequestSnapshot]>([]) + private let didFail = Locked(false) + + func respond( + within session: LanguageModelSession, + to prompt: Prompt, + generating type: Content.Type, + includeSchemaInPrompt: Bool, + options: GenerationOptions + ) async throws -> LanguageModelSession.Response where Content: Generable { + let context = session.resolvedRequestContext() + snapshots.withLock { + $0.append( + .init( + instructions: context.instructions?.description, + tools: context.tools.map(\.name) + ) + ) + } + + let shouldFail = didFail.withLock { didFail in + defer { didFail = true } + return !didFail + } + if shouldFail { + let tool = try #require(context.tools.first) + _ = try await tool.makeOutputSegments(from: GeneratedContent(properties: [:])) + state.select(.b) + throw DynamicFixtureError.failed + } + + let raw = GeneratedContent("Done") + return .init(content: try Content(raw), rawContent: raw, transcriptEntries: []) + } + + func streamResponse( + within session: LanguageModelSession, + to prompt: Prompt, + generating type: Content.Type, + includeSchemaInPrompt: Bool, + options: GenerationOptions + ) -> sending LanguageModelSession.ResponseStream where Content: Generable { + let stream = AsyncThrowingStream.Snapshot, any Error> { + $0.finish(throwing: DynamicFixtureError.failed) + } + return .init(stream: stream) + } +} + +private final class DynamicCancellationControl: @unchecked Sendable { + let started: AsyncStream + + private let startedContinuation: AsyncStream.Continuation + private let cancellationContinuation = Locked?>(nil) + + init() { + (started, startedContinuation) = AsyncStream.makeStream() + } + + func signalStarted() { + startedContinuation.yield(()) + } + + func waitForCancellation() async throws { + try await withTaskCancellationHandler { + try await withCheckedThrowingContinuation { continuation in + let isAlreadyCancelled = cancellationContinuation.withLock { stored in + guard !Task.isCancelled else { return true } + stored = continuation + return false + } + if isAlreadyCancelled { + continuation.resume(throwing: CancellationError()) + } + } + } onCancel: { + let continuation = cancellationContinuation.withLock { stored in + defer { stored = nil } + return stored + } + continuation?.resume(throwing: CancellationError()) + } + } +} + +private struct CancellingAfterToolModel: LanguageModel { + typealias UnavailableReason = Never + + let state: DynamicFixtureState + let control: DynamicCancellationControl + let snapshots = Locked<[DynamicRequestSnapshot]>([]) + private let didSuspend = Locked(false) + + func respond( + within session: LanguageModelSession, + to prompt: Prompt, + generating type: Content.Type, + includeSchemaInPrompt: Bool, + options: GenerationOptions + ) async throws -> LanguageModelSession.Response where Content: Generable { + let context = session.resolvedRequestContext() + snapshots.withLock { + $0.append( + .init( + instructions: context.instructions?.description, + tools: context.tools.map(\.name) + ) + ) + } + + let shouldSuspend = didSuspend.withLock { didSuspend in + defer { didSuspend = true } + return !didSuspend + } + if shouldSuspend { + let tool = try #require(context.tools.first) + _ = try await tool.makeOutputSegments(from: GeneratedContent(properties: [:])) + state.select(.b) + control.signalStarted() + try await control.waitForCancellation() + } + + let raw = GeneratedContent("Done") + return .init(content: try Content(raw), rawContent: raw, transcriptEntries: []) + } + + func streamResponse( + within session: LanguageModelSession, + to prompt: Prompt, + generating type: Content.Type, + includeSchemaInPrompt: Bool, + options: GenerationOptions + ) -> sending LanguageModelSession.ResponseStream where Content: Generable { + let stream = AsyncThrowingStream.Snapshot, any Error> { + $0.finish(throwing: CancellationError()) + } + return .init(stream: stream) + } +} diff --git a/Tests/AnyLanguageModelTests/SystemLanguageModelTests.swift b/Tests/AnyLanguageModelTests/SystemLanguageModelTests.swift index 51168808..44690e50 100644 --- a/Tests/AnyLanguageModelTests/SystemLanguageModelTests.swift +++ b/Tests/AnyLanguageModelTests/SystemLanguageModelTests.swift @@ -92,6 +92,50 @@ import Testing #expect(schema.defs[nestedTypeName] != nil) } + private final class SwitchState: @unchecked Sendable { + var useWeather = false + } + + private struct SwitchedInstructions: DynamicInstructions { + let state: SwitchState + + var body: some DynamicInstructions { + if state.useWeather { + Instructions("Answer weather questions with the getWeather tool.") + WeatherTool() + } else { + Instructions("Reply with exactly one word: apple.") + } + } + } + + private struct BriefInstructions: DynamicInstructions { + var body: some DynamicInstructions { + Instructions("Be brief.") + } + } + + /// Before OS 27, and on tvOS, the system model rejects dynamic instructions + /// before it uses the model, so this runs even where the model is unavailable. + @available(macOS 26.0, iOS 26.0, tvOS 26.0, visionOS 26.0, *) + @Test func dynamicInstructionsAreUnavailableWithoutNativeSupport() async throws { + var hasNativeSupport = false + #if compiler(>=6.4) && !os(tvOS) + if #available(macOS 27.0, iOS 27.0, visionOS 27.0, *) { + hasNativeSupport = true + } + #endif + guard !hasNativeSupport else { return } + + let session = LanguageModelSession(model: SystemLanguageModel(), dynamicInstructions: BriefInstructions()) + await #expect(throws: SystemLanguageModel.Error.dynamicInstructionsUnavailable) { + try await session.respond(to: "Hello") + } + await #expect(throws: SystemLanguageModel.Error.dynamicInstructionsUnavailable) { + for try await _ in session.streamResponse(to: "Hello") {} + } + } + @Suite( "SystemLanguageModel", .enabled(if: isSystemLanguageModelAvailable) @@ -199,6 +243,32 @@ import Testing #expect(!snapshots.last!.rawContent.jsonString.isEmpty) } + #if compiler(>=6.4) && !os(tvOS) + @available(macOS 26.0, iOS 26.0, tvOS 26.0, visionOS 26.0, *) + @Test func dynamicInstructionsChangeBetweenRequests() async throws { + guard #available(macOS 27.0, iOS 27.0, visionOS 27.0, *) else { return } + let state = SwitchState() + let session = LanguageModelSession( + model: SystemLanguageModel(), + dynamicInstructions: SwitchedInstructions(state: state) + ) + let options = GenerationOptions(sampling: .greedy) + + let first = try await session.respond(to: "What should you reply?", options: options) + #expect(first.content.localizedCaseInsensitiveContains("apple")) + + state.useWeather = true + let second = try await session.respond(to: "How's the weather in San Francisco?", options: options) + #expect(second.content.contains("72°F")) + + #expect( + !session.transcript.contains { + if case .instructions = $0 { true } else { false } + } + ) + } + #endif + @available(macOS 26.0, iOS 26.0, tvOS 26.0, visionOS 26.0, *) @Test func withTools() async throws { let weatherTool = WeatherTool()