diff --git a/README.md b/README.md index 9714430..877685b 100644 --- a/README.md +++ b/README.md @@ -284,7 +284,7 @@ Agents depend only on `ILlmClient`; `LlmClientFactory` resolves a keyed implemen | `OpenAICompatible` (aliases `Ollama`, `vLLM`, `Local`) | Any `/v1/chat/completions` server via the OpenAI SDK — self-hosted / air-gapped, your own prices | `Modules.Llm` | | `MAF` | Microsoft Agent Framework over Azure OpenAI | `Modules.Llm` | | `RemoteAgent` | SignalR-dispatched dev-machine agent | `Modules.RemoteAgent` | -| `Offline` | Keyless canned output — only at the end of a failover chain when offline fallback is on | `Modules.Llm` | +| `Offline` | Keyless canned output — only at the end of a failover chain when offline fallback is on, and never for Board agent runs | `Modules.Llm` | Routing per call: `Llm:ForceProvider` (if set) → the agent profile's provider → `Llm:Provider`. `Llm:Fallbacks` adds inter-provider failover (e.g. Claude → AzureOpenAI) above each provider's own diff --git a/docs/CONFIG.md b/docs/CONFIG.md index 6fd56ea..9043b64 100644 --- a/docs/CONFIG.md +++ b/docs/CONFIG.md @@ -78,7 +78,7 @@ Defaults below are the code defaults (tier 5). **Scope**: *Tenant* = each tenant |---|---|---|---| | `Llm:Provider` | `Claude` | Tenant | Used by any agent whose profile names no provider. `Anthropic` is an alias of `Claude`. | | `Llm:ForceProvider` | empty | Tenant | Route every agent to one provider. | -| `Llm:OfflineFallback` | **`true` in Development, else `false`** | Platform | Keyless provider → canned, schema-valid output, labelled Offline everywhere (runs, Workflow chip, PR refusal). Lets a fresh checkout run with no secrets. Locked off in Production. | +| `Llm:OfflineFallback` | **`true` in Development, else `false`** | Platform | Keyless provider → canned, schema-valid output, labelled Offline everywhere (runs, Workflow chip, PR refusal). Lets a fresh checkout run with no secrets. Locked off in Production. Never applies to Board agent runs (New idea, Coding, Reviewer, Testing, the Autopilot lead, Retrospective): those act on a real project, so they fail with the real reason (e.g. a plan limit) instead of getting canned text. | | `Llm:Claude:ApiKey` / `ApiKeys` | empty | Tenant | Secret, write-only in the UI. Pool = round-robin with 429 failover. | | `Llm:AzureOpenAi:Endpoint` / `ApiKey` / `ApiKeys` | empty | Tenant | Endpoint must be absolute `https://`. | | `Llm:AzureOpenAi:Model` | `gpt-4.1` | Tenant | Default deployment. | diff --git a/src/AgentOs.Domain/Llm/LlmCallScope.cs b/src/AgentOs.Domain/Llm/LlmCallScope.cs index a38e631..97e780b 100644 --- a/src/AgentOs.Domain/Llm/LlmCallScope.cs +++ b/src/AgentOs.Domain/Llm/LlmCallScope.cs @@ -25,7 +25,9 @@ public sealed record LlmCallRecord( string? Error, DateTimeOffset At); -/// Collects the LLM calls made in the current async flow while it is open (innermost scope wins). +/// Collects the LLM calls made in the current async flow while it is open (innermost scope wins). An open +/// scope also marks the flow as a Board agent run on a real project: the gateway never lets the keyless offline demo +/// answer in place of the real model there (see LlmClientFactory). public sealed class LlmCallScope : IDisposable { private static readonly AsyncLocal CurrentScope = new(); diff --git a/src/AgentOs.Modules.Llm/LlmClientFactory.cs b/src/AgentOs.Modules.Llm/LlmClientFactory.cs index f4a6b5a..ea81485 100644 --- a/src/AgentOs.Modules.Llm/LlmClientFactory.cs +++ b/src/AgentOs.Modules.Llm/LlmClientFactory.cs @@ -61,9 +61,13 @@ public ILlmClient Create(string providerName) // to the END of the chain so a no-key LlmException falls through to canned schema-valid output instead // of failing the run. Skipped when the effective provider already IS Offline, or it's already a // configured fallback, so it never appears twice. - var offlineFallback = _settings is null + // Never for a Board agent run (an open LlmCallScope: Discovery, Coding, Reviewer, Testing, the Autopilot lead, + // Retrospective): those write tickets, reviews and merges on a real project, so canned demo text standing in + // for a model that hit its plan limit or went offline would become junk tickets or a false verdict. They fail + // with the real reason instead ("session limit · resets 23:50") and the Board says so. + var offlineFallback = AgentOs.Domain.Llm.LlmCallScope.Current is null && (_settings is null ? _options.OfflineFallback - : LiveSettings.ReadBool(_settings, LlmSettings.OfflineFallback, tenant, _options.OfflineFallback); + : LiveSettings.ReadBool(_settings, LlmSettings.OfflineFallback, tenant, _options.OfflineFallback)); if (offlineFallback && !string.Equals(NormalizeKey(effective), OfflineLlmClient.ProviderName, StringComparison.Ordinal)) { diff --git a/src/AgentOs.Web/Components/Pages/BoardApp.razor b/src/AgentOs.Web/Components/Pages/BoardApp.razor index 0aa3acc..24ae8a1 100644 --- a/src/AgentOs.Web/Components/Pages/BoardApp.razor +++ b/src/AgentOs.Web/Components/Pages/BoardApp.razor @@ -1599,7 +1599,7 @@ else if (_tab == Tab.Sessions) } _ideaStep = IdeaStep.Review; }, - ex => _ideaError = $"Generation failed: {ex.Message}"); + ex => _ideaError = $"The AI couldn't write the spec, so nothing was created — {ex.Message} Try again once it is back."); _ideaBusy = false; } diff --git a/tests/AgentOs.Tests/Llm/LlmClientFactoryLiveSettingsTests.cs b/tests/AgentOs.Tests/Llm/LlmClientFactoryLiveSettingsTests.cs index b8deb1b..172700a 100644 --- a/tests/AgentOs.Tests/Llm/LlmClientFactoryLiveSettingsTests.cs +++ b/tests/AgentOs.Tests/Llm/LlmClientFactoryLiveSettingsTests.cs @@ -12,6 +12,7 @@ using AgentOs.SharedKernel.Modularity; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; +using NSubstitute; using Shouldly; using Xunit; @@ -91,6 +92,37 @@ public async Task Create_PlatformOfflineFallback_TogglesTheCannedSafetyNetLive() (await settings.SetAsync(LlmSettings.OfflineFallback, "false", SettingScope.Tenant, "acme", "admin")).Ok.ShouldBeFalse(); } + [Fact] + public async Task Create_InsideBoardAgentRun_NeverFallsOverToTheOfflineDemo() + { + // 2026-10-23 on the lab: the member's Claude plan hit its session limit, the Offline client answered New idea + // with its canned spec, and three junk tickets landed on a real repo. A Board agent run must fail instead. + var services = new ServiceCollection(); + services.AddLogging(); + var config = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary { ["Llm:Provider"] = "Claude", ["Llm:OfflineFallback"] = "true" }) + .Build(); + services.AddModulesFromAssemblies(config, typeof(LlmModule).Assembly, typeof(AgentOs.Modules.AppConfig.AppConfigModule).Assembly); + var limited = Substitute.For(); + limited.Provider.Returns("Limited"); + limited.SendAsync(Arg.Any(), Arg.Any()) + .Returns>(_ => throw new LlmException("You've hit your session limit · resets 11:50pm")); + services.AddKeyedSingleton("Limited", limited); + var factory = services.BuildServiceProvider().GetRequiredService(); + var request = new LlmRequest("You are the Requirement Agent of an SDLC pipeline.", "An idea", "claude-sonnet-5", MaxTokens: 1000); + using var _ = AmbientIdentity.Push("acme", "u"); + + // Outside a Board run (Workflow demo, standalone) the canned safety net still answers. + (await factory.Create("Limited").SendAsync(request)).Provider.ShouldBe(OfflineLlmClient.ProviderName); + + using (var run = LlmCallScope.Begin()) + { + var ex = await Should.ThrowAsync(() => factory.Create("Limited").SendAsync(request)); + ex.Message.ShouldContain("session limit"); + run.Calls.ShouldHaveSingleItem().Success.ShouldBeFalse(); // Insights still sees the failed call + } + } + // FailoverLlmClient is internal; a chain reports the primary's provider, so detect it by type name. private static bool IsChain(ILlmClient client) => client.GetType().Name == "FailoverLlmClient";