From 1110473fa321397b2cda9a409b891d5952b00196 Mon Sep 17 00:00:00 2001 From: "mars.yu" <18671933+yuyixg@users.noreply.github.com> Date: Wed, 18 Nov 2026 14:54:39 +0800 Subject: [PATCH 1/3] feat(twilio): a relay session whose playback is stuck is connected again and says what never played On some calls Twilio said the agent started speaking and never that it stopped. Nothing sent after that played, none of the caller's words came through, and Twilio reported no error, until the caller hung up (2 of 15 calls with a full closing on an RSP dev line in 40 hours). The session now reads Twilio's speaker events (the events attribute set to speaker-events) and can recover: - ConversationRelaySetting.StuckPlaybackSeconds, also per agent: how long past the moment everything said should have finished playing (estimated from its length, on the slow side) Twilio may still say the agent is speaking, with nothing cut into and no turn running, before the playback is taken as stuck. 0, the default, turns it off. - Stuck, the session ends with handoffData {"action":"reconnect","resume":{...}}: the lines said since Twilio last said the playback stopped (fillers and idle prompts left out), each with whether it went out to play to the end, and the flow's quiet wait and goodbye on silence. Only while the call may be connected again (MaxReconnects): the session reads its count from a new "reconnects" Parameter. - The action callback answers it with a new carrying the resume as a "resume" Parameter, and tells the hooks nothing, as the call goes on. A call that can't be connected again ends as the agent's. - The new session says those lines again, each as it went out (they are in the history already, so not kept again), and takes the quiet wait and goodbye on silence back. - While it watches, no idle prompt is said while Twilio still plays (it would only queue behind a stuck playback and put off finding it), the caller's silence starts no sooner than Twilio says the playback ended, and what goes out after "agentSpeaking off" or a cut-in plays from then, not after what the estimate still had to come. Tests are in onebrain (UnitTest/TwilioHooks/ConversationRelayWatchdogTests.StuckPlayback.cs), which runs this plugin's relay session over a fake socket: 71 of 71 relay tests pass, 10 of them new. Co-Authored-By: Claude Opus 5.5 --- .../TwilioConversationRelayController.cs | 40 ++++ .../Models/Relay/ConversationRelayHandoff.cs | 11 + .../Relay/ConversationRelayParameters.cs | 10 + .../Models/Relay/ConversationRelayResume.cs | 37 ++++ .../Services/ConversationRelaySession.cs | 199 ++++++++++++++++-- .../Services/TwilioService.cs | 11 +- .../Settings/ConversationRelayAgentSetting.cs | 1 + .../Settings/ConversationRelaySetting.cs | 15 ++ 8 files changed, 310 insertions(+), 14 deletions(-) create mode 100644 src/Plugins/BotSharp.Plugin.Twilio/Models/Relay/ConversationRelayResume.cs diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioConversationRelayController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioConversationRelayController.cs index 294b37077..1111bf937 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioConversationRelayController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioConversationRelayController.cs @@ -52,6 +52,23 @@ await HookEmitter.Emit(_services, var callStillUp = !string.Equals(request.CallStatus, "completed", StringComparison.OrdinalIgnoreCase); var canReconnect = request.Reconnects < _settings.ConversationRelay.MaxReconnects; + + // The session's playback was stuck: a new session says what never finished playing. Nothing is told to the hooks: + // the call goes on. + if (action == ConversationRelayHandoff.Reconnect) + { + if (callStillUp && canReconnect) + { + _logger.LogWarning($"Reconnecting ConversationRelay for conversation {request.ConversationId}: its playback " + + $"was stuck (attempt {request.Reconnects + 1})."); + return TwiML(twilio.ReturnConversationRelay(request.AgentId, request.ConversationId, welcomeGreeting: null, + request.Reconnects + 1, GetHandoffField(request.HandoffData, ConversationRelayHandoff.ResumeField))); + } + // The session asks only while it may (it knows its reconnects): a caller who hung up meanwhile ends the call + // as any hang-up does, and anything else as the agent's + action = callStillUp ? ConversationRelayHandoff.HangUp : null; + } + if (request.SessionStatus == "failed" && callStillUp && canReconnect) { _logger.LogWarning($"Reconnecting ConversationRelay for conversation {request.ConversationId} " + @@ -103,4 +120,27 @@ await HookEmitter.Emit(_services, return handoffData; } } + + /// + /// A field of the handoffData as JSON, or null when it has none + /// + private static string? GetHandoffField(string? handoffData, string name) + { + if (string.IsNullOrWhiteSpace(handoffData)) + { + return null; + } + + try + { + using var doc = JsonDocument.Parse(handoffData); + return doc.RootElement.ValueKind == JsonValueKind.Object && doc.RootElement.TryGetProperty(name, out var field) + ? field.GetRawText() + : null; + } + catch (JsonException) + { + return null; + } + } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/Relay/ConversationRelayHandoff.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/Relay/ConversationRelayHandoff.cs index 1338b10df..005a80b76 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Models/Relay/ConversationRelayHandoff.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/Relay/ConversationRelayHandoff.cs @@ -7,4 +7,15 @@ public static class ConversationRelayHandoff { public const string HangUp = "hangup"; public const string Transfer = "transfer"; + + /// + /// The session's playback was stuck: the call is connected to a new session, which says what never finished playing + /// (ConversationRelaySetting.StuckPlaybackSeconds). The handoffData carries it under . + /// + public const string Reconnect = "reconnect"; + + /// + /// The handoffData field a carries its in + /// + public const string ResumeField = "resume"; } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/Relay/ConversationRelayParameters.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/Relay/ConversationRelayParameters.cs index b8b78f40c..294b0c531 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Models/Relay/ConversationRelayParameters.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/Relay/ConversationRelayParameters.cs @@ -6,4 +6,14 @@ namespace BotSharp.Plugin.Twilio.Models.Relay; public static class ConversationRelayParameters { public const string WelcomeGreetingLength = "welcome-greeting-length"; + + /// + /// How many times the call's relay has been connected again + /// + public const string Reconnects = "reconnects"; + + /// + /// What a session connected again in place of one whose playback was stuck starts with () + /// + public const string Resume = "resume"; } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/Relay/ConversationRelayResume.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/Relay/ConversationRelayResume.cs new file mode 100644 index 000000000..3c1c29005 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/Relay/ConversationRelayResume.cs @@ -0,0 +1,37 @@ +namespace BotSharp.Plugin.Twilio.Models.Relay; + +/// +/// What a session connected again in place of one whose playback was stuck starts with: the lines the caller never heard +/// all of, and how the flow last said to treat a silent caller +/// +/// +/// It goes from the stuck session to the new one through Twilio (the "end" message's handoffData, then a TwiML Parameter), +/// so the new session may run on another instance. +/// +public class ConversationRelayResume +{ + /// + /// The lines said since Twilio last said playback stopped, in order, but fillers and idle prompts + /// + public List Lines { get; set; } = []; + + /// + /// The flow was waiting quietly for the caller (ConversationRelayStates.QuietWait) + /// + public bool QuietWait { get; set; } + + /// + /// What the flow said to say if the caller stays silent until the hang-up (ConversationRelayStates.GoodbyeOnSilence) + /// + public string? GoodbyeOnSilence { get; set; } +} + +public class ConversationRelayResumeLine +{ + public string Text { get; set; } = string.Empty; + + /// + /// It went out to play to the end whoever speaks over it, and goes out so again + /// + public bool KeepUninterrupted { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/ConversationRelaySession.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/ConversationRelaySession.cs index 1bac875b5..5405e9006 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/ConversationRelaySession.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/ConversationRelaySession.cs @@ -134,10 +134,21 @@ public class ConversationRelaySession private bool _lastLineHeardSome; // What was said, newest last, with _saidCount as it went out: tells which utterance Twilio says was playing at a - // cut-in. Only the last few are kept. - private readonly List<(string Text, int Mark)> _utterances = []; + // cut-in, and which never finished playing when the playback got stuck (with whether it went out to play to the end, + // and whether it is one to say again then: not a filler or an idle prompt). Only the last few are kept. + private readonly List<(string Text, int Mark, bool KeptUninterrupted, bool Again)> _utterances = []; private const int UtterancesKept = 8; + // Twilio's speaker events (the events attribute with speaker-events): whether it last said the agent is speaking, and + // _saidCount when it last said the playback stopped (all of it played, or the caller cut in). What was said after that + // isn't known to have been heard. + private bool _agentSpeaking; + private int _playedThroughMark; + // When everything said should have finished playing, by its length (EstimateSpeech) + private DateTime _playedByUtc = DateTime.MinValue; + // How many times the call's relay has been connected again (ConversationRelayParameters.Reconnects) + private int _reconnects; + // The line the caller cut into, to say again once nothing follows the cut-in (ResumeAfterCutInSeconds) private string? _resumeLine; private DateTime _cutInUtc; @@ -265,17 +276,58 @@ private async Task HandleMessageAsync(string json) _to = GetString(root, "to"); _direction = GetString(root, "direction"); var greetingLength = 0; - if (root.TryGetProperty("customParameters", out var parameters) - && GetString(parameters, ConversationRelayParameters.WelcomeGreetingLength) is string length) + var reconnects = 0; + string? resume = null; + if (root.TryGetProperty("customParameters", out var parameters)) { - int.TryParse(length, out greetingLength); + if (GetString(parameters, ConversationRelayParameters.WelcomeGreetingLength) is string length) + { + int.TryParse(length, out greetingLength); + } + if (GetString(parameters, ConversationRelayParameters.Reconnects) is string count) + { + int.TryParse(count, out reconnects); + } + resume = GetString(parameters, ConversationRelayParameters.Resume); } lock (_sync) { _quietSinceUtc = DateTime.UtcNow + EstimateSpeech(greetingLength); + _playedByUtc = _quietSinceUtc; + _reconnects = reconnects; } _logger.LogInformation( $"ConversationRelay session started for conversation {_conversationId} (call {_callSid})."); + if (resume != null) + { + await ResumeAsync(resume); + } + break; + + case "info": + // Twilio's speaker events. One "on" and one "off" cover everything played back to back, however many lines + // went out meanwhile: at "off" all of it has played. + if (GetString(root, "name") == "agentSpeaking") + { + lock (_sync) + { + _agentSpeaking = GetString(root, "value") == "on"; + if (!_agentSpeaking) + { + _playedThroughMark = _saidCount; + // Nothing is left to play: what goes out next plays from now, not after what the estimate + // still had to come + var now = DateTime.UtcNow; + _playedByUtc = now; + // Watching for a stuck playback, the caller's silence starts no sooner than the playback ended + // (the estimate may be short of it): no idle prompt was said while it played (WatchIdleAsync) + if (_relay.StuckPlaybackSeconds > 0 && _quietSinceUtc != DateTime.MaxValue && _quietSinceUtc < now) + { + _quietSinceUtc = now; + } + } + } + } break; case "prompt": @@ -334,6 +386,11 @@ private async Task HandleMessageAsync(string json) lock (_sync) { _quietSinceUtc = DateTime.UtcNow; + // The playback stopped where the caller cut in: what is said again after a cut-in is the cut-in's to + // decide (ResumeAfterCutInSeconds), not a stuck playback's + _agentSpeaking = false; + _playedThroughMark = _saidCount; + _playedByUtc = DateTime.UtcNow; // The caller talked over a reply still being streamed: don't send the rest of it. A stream already // dropped (the turn timed out and the apology is playing, or the caller cut in before) or a @@ -772,7 +829,7 @@ private async Task SpeakFillerAsync(int seqNumber, string text, TimeSpan i { return false; } - RememberUtterance(text, _saidCount + 1); + RememberUtterance(text, _saidCount + 1, again: false); // Noted as it goes out, under the send lock: a turn that fails meanwhile finds every filler said before // its apology callbacks.NoteFillerSaid(text); @@ -806,13 +863,16 @@ private void NoteLine(string? line, DateTime? sentUtc = null, bool keepHeard = f /// /// Notes something said, with the _saidCount it went out at. Call under . /// - private void RememberUtterance(string text, int mark) + /// It went out to play to the end whoever speaks over it + /// Said again on a new session if it never finished playing (ReconnectIfStuckAsync): not for a + /// filler or an idle prompt, which were about a moment that has passed + private void RememberUtterance(string text, int mark, bool keptUninterrupted = false, bool again = true) { if (string.IsNullOrWhiteSpace(text)) { return; } - _utterances.Add((text.Trim(), mark)); + _utterances.Add((text.Trim(), mark, keptUninterrupted, again)); if (_utterances.Count > UtterancesKept) { _utterances.RemoveAt(0); @@ -862,6 +922,10 @@ private async Task WatchIdleAsync(CancellationToken cancellationToken) while (!cancellationToken.IsCancellationRequested) { await Task.Delay(TimeSpan.FromMilliseconds(500), cancellationToken); + if (await ReconnectIfStuckAsync()) + { + return; + } await ResumeIfSilentAsync(); var askAgain = false; @@ -874,7 +938,11 @@ private async Task WatchIdleAsync(CancellationToken cancellationToken) lock (_sync) { var quietFor = DateTime.UtcNow - _quietSinceUtc; - if (_ended || _isProcessing || _quietSinceUtc == DateTime.MaxValue || _resumeLine != null + // Watching for a stuck playback, a caller Twilio is still playing to isn't silent: an idle prompt then would + // only queue behind the playback, and each one would put off finding it stuck until the idle prompts ran out + // and hung up (ReconnectIfStuckAsync) + var stillPlaying = _relay.StuckPlaybackSeconds > 0 && _agentSpeaking; + if (_ended || _isProcessing || _quietSinceUtc == DateTime.MaxValue || _resumeLine != null || stillPlaying || quietFor < IdleTimeout(_idlePrompts, _quietWait)) { continue; @@ -951,6 +1019,107 @@ await SayOwnLineAsync(PromptText($"say-it-again-{Random.Shared.Next(1, 5)}", Def } } + /// + /// Ends a session whose playback is stuck (ConversationRelaySetting.StuckPlaybackSeconds) and has Twilio connect the call + /// again: the new session says the lines that never finished playing (). Whether it did. + /// + /// + /// Stuck: Twilio still says the agent is speaking well after everything said should have finished playing, and no turn + /// is running (one still running may say more, which then goes with the rest). Only while the call may still be + /// connected again (MaxReconnects): past that the action callback would hang it up. + /// Why say the lines again rather than carry on: a stuck playback plays nothing queued behind it and lets none of the + /// caller's words through, so the caller heard neither the lines nor anything after them (dev calls 1c28cefa, 46bc439c). + /// + private async Task ReconnectIfStuckAsync() + { + ConversationRelayResume resume; + double overdueSeconds; + lock (_sync) + { + var stuckAfter = _relay.StuckPlaybackSeconds; + if (stuckAfter <= 0 || _ended || _isProcessing || !_agentSpeaking || _reconnects >= _relay.MaxReconnects) + { + return false; + } + var overdue = DateTime.UtcNow - _playedByUtc; + if (overdue < TimeSpan.FromSeconds(stuckAfter)) + { + return false; + } + + var lines = _utterances.Where(u => u.Mark > _playedThroughMark && u.Again) + .Select(u => new ConversationRelayResumeLine { Text = u.Text, KeepUninterrupted = u.KeptUninterrupted }) + .ToList(); + // Nothing to say again went out since the playback last stopped (it went out just as Twilio said so): the last + // line, which the caller may not have heard either + if (lines.Count == 0 && _lastLine != null) + { + lines.Add(new ConversationRelayResumeLine { Text = _lastLine, KeepUninterrupted = _lastLineKeptUninterrupted }); + } + resume = new ConversationRelayResume { Lines = lines, QuietWait = _quietWait, GoodbyeOnSilence = _goodbyeOnSilence }; + overdueSeconds = overdue.TotalSeconds; + // Ended in the same step, so caller input arriving now can't start a turn on a session being replaced + _ended = true; + } + + _logger.LogWarning($"ConversationRelay playback is stuck in conversation {_conversationId}: Twilio still says the " + + $"agent is speaking {overdueSeconds:0.0} s after all of it should have played. Connecting the call again to say " + + $"{resume.Lines.Count} line(s) again."); + await SendAsync(new + { + type = "end", + handoffData = JsonSerializer.Serialize(new Dictionary + { + ["action"] = ConversationRelayHandoff.Reconnect, + [ConversationRelayHandoff.ResumeField] = resume + }) + }); + return true; + } + + /// + /// Starts a session that takes the place of one whose playback was stuck (): the + /// lines the caller never heard all of are said again, each as it went out, and a silent caller is treated as the flow + /// last said. They are in the history already, so they aren't kept again. + /// + private async Task ResumeAsync(string json) + { + ConversationRelayResume? resume; + try + { + resume = JsonSerializer.Deserialize(json); + } + catch (JsonException ex) + { + _logger.LogWarning($"Unreadable resume for conversation {_conversationId}; nothing is said again: {ex.Message}"); + return; + } + if (resume == null) + { + return; + } + + lock (_sync) + { + _quietWait = resume.QuietWait; + _goodbyeOnSilence = resume.GoodbyeOnSilence; + } + var said = 0; + foreach (var line in resume.Lines) + { + if (await SpeakAsync(line.Text, canSpeak: () => + { + NoteLine(line.Text, keptUninterrupted: line.KeepUninterrupted); + return true; + }, keepUninterrupted: line.KeepUninterrupted)) + { + said++; + } + } + _logger.LogInformation($"ConversationRelay session for conversation {_conversationId} took the place of one whose " + + $"playback was stuck; said {said} line(s) again."); + } + // How long the caller may stay silent before the idle prompt after this many (the last is before the hang-up) private TimeSpan IdleTimeout(int prompts, bool quietWait) { @@ -1087,7 +1256,7 @@ private async Task KeepFillersLeftAsync(AgentTurnCallbacks callbacks) /// Extra condition, checked under once the send lock is held private async Task SayOwnLineAsync(string text, string label, Func? canSpeak = null) { - if (await SpeakAsync(text, canSpeak)) + if (await SpeakAsync(text, canSpeak, sayAgainIfStuck: label != ConversationRelayLabels.IdlePrompt)) { await KeepSaidAsync(text, label, DateTime.UtcNow); } @@ -1179,7 +1348,7 @@ await SendIfAsync(() => _lastStreamSegment = _streamSegment.ToString(); _streamSegment.Clear(); MarkSpeaking(EstimateSpeech(_lastStreamSegment.Length)); - RememberUtterance(_lastStreamSegment, _utteranceStartMark); + RememberUtterance(_lastStreamSegment, _utteranceStartMark, keep); // Cut into while it streamed: the caller's words are about it, so it dates from when it started and keeps // what they heard of it, not from now (a "Yes." said over it was dropped as a tail, review) if (_utteranceCutIn) @@ -1373,8 +1542,10 @@ await SpeakIfAsync(() => !_ended && (canSpeak?.Invoke() ?? true), /// Extra condition, checked under once the send lock is held /// The caller must hear all of it, so it plays to the end whoever speaks over it + /// Said again on a new session if it never finished playing (RememberUtterance) /// Whether it went out - private async Task SpeakAsync(string? text, Func? canSpeak = null, bool keepUninterrupted = false) + private async Task SpeakAsync(string? text, Func? canSpeak = null, bool keepUninterrupted = false, + bool sayAgainIfStuck = true) { if (string.IsNullOrWhiteSpace(text)) { @@ -1387,7 +1558,7 @@ private async Task SpeakAsync(string? text, Func? canSpeak = null, b { return false; } - RememberUtterance(text, _saidCount + 1); + RememberUtterance(text, _saidCount + 1, keepUninterrupted, sayAgainIfStuck); return true; }, TextFrame(text, last: true, keepUninterrupted), EstimateSpeech(text.Length)); @@ -1560,6 +1731,8 @@ private void MarkSpeaking(TimeSpan duration) var now = DateTime.UtcNow; var start = _quietSinceUtc == DateTime.MaxValue || _quietSinceUtc < now ? now : _quietSinceUtc; _quietSinceUtc = start + duration; + // Queued behind what is still to play; unlike the silence clock, the caller's words don't move it + _playedByUtc = (_playedByUtc < now ? now : _playedByUtc) + duration; } private static TimeSpan EstimateSpeech(int characters) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs index 6078d950f..7ec986324 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs @@ -272,8 +272,11 @@ public VoiceResponse ReturnBidirectionalMediaStreamsInstructions(ConversationalV /// Hands the call to Twilio ConversationRelay. Twilio opens a WebSocket to twilio/relay/{agentId}/{conversationId} /// and posts to twilio/relay/action when the session ends. /// + /// How many times the call's relay has been connected again + /// What the session starts with when it takes the place of one whose playback was stuck: a + /// as JSON public VoiceResponse ReturnConversationRelay( - string agentId, string conversationId, string? welcomeGreeting, int reconnects = 0) + string agentId, string conversationId, string? welcomeGreeting, int reconnects = 0, string? resume = null) { var relay = _settings.ConversationRelay.For(agentId); var query = $"agent-id={Uri.EscapeDataString(agentId)}" + @@ -309,6 +312,12 @@ public VoiceResponse ReturnConversationRelay( // Lets the session time its idle check from the end of the greeting var greetingLength = (welcomeGreeting?.Length ?? 0).ToString(); element.Parameter(name: ConversationRelayParameters.WelcomeGreetingLength, value: greetingLength); + // The session reconnects a stuck playback only while the call may still be connected again + element.Parameter(name: ConversationRelayParameters.Reconnects, value: reconnects.ToString()); + if (!string.IsNullOrWhiteSpace(resume)) + { + element.Parameter(name: ConversationRelayParameters.Resume, value: resume); + } connect.Append(element); var response = new VoiceResponse(); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Settings/ConversationRelayAgentSetting.cs b/src/Plugins/BotSharp.Plugin.Twilio/Settings/ConversationRelayAgentSetting.cs index 67a9823c5..7171a223e 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Settings/ConversationRelayAgentSetting.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Settings/ConversationRelayAgentSetting.cs @@ -23,4 +23,5 @@ public class ConversationRelayAgentSetting public int? MaxQuietWaitPrompts { get; set; } public double? ResumeAfterCutInSeconds { get; set; } public bool? DropBackchannelTails { get; set; } + public double? StuckPlaybackSeconds { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Settings/ConversationRelaySetting.cs b/src/Plugins/BotSharp.Plugin.Twilio/Settings/ConversationRelaySetting.cs index 5ce5fb7c5..9cb143d0c 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Settings/ConversationRelaySetting.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Settings/ConversationRelaySetting.cs @@ -144,6 +144,20 @@ public class ConversationRelaySetting /// public bool DropBackchannelTails { get; set; } + /// + /// Seconds past the moment everything said should have finished playing, with Twilio still saying the agent is speaking + /// and nothing cut into, before the session's playback is taken as stuck: the session is ended, Twilio connects the + /// call again, and the lines that never finished playing are said on the new session. Needs Twilio's speaker events + /// (the events attribute with speaker-events); stops after . 0 turns it off. + /// + /// + /// Why: on some calls Twilio said the agent started speaking and never that it stopped. Nothing said after that played, + /// and none of the caller's words came through, until they hung up; Twilio reported no error (dev calls 1c28cefa, + /// 46bc439c). How long a line takes is estimated from its length, on the slow side, so this is how much later than + /// even that the playback must still be going. + /// + public double StuckPlaybackSeconds { get; set; } + /// /// Settings for one agent's calls, by agent id, over the ones above /// @@ -172,6 +186,7 @@ public ConversationRelaySetting For(string? agentId) settings.MaxQuietWaitPrompts = agent.MaxQuietWaitPrompts ?? MaxQuietWaitPrompts; settings.ResumeAfterCutInSeconds = agent.ResumeAfterCutInSeconds ?? ResumeAfterCutInSeconds; settings.DropBackchannelTails = agent.DropBackchannelTails ?? DropBackchannelTails; + settings.StuckPlaybackSeconds = agent.StuckPlaybackSeconds ?? StuckPlaybackSeconds; return settings; } From 3350d2347fb0acf561664b93e6f6b487ef1d6ff0 Mon Sep 17 00:00:00 2001 From: "mars.yu" <18671933+yuyixg@users.noreply.github.com> Date: Wed, 18 Nov 2026 15:22:30 +0800 Subject: [PATCH 2/3] fix(twilio): a stuck playback drops the relay socket instead of ending the session Twilio acts on an "end" message only once what is queued has played: on dev, every session we ended with a goodbye and "end" sent together was reported ended 5.4 to 6.0 s later, the goodbye's length (9 calls). A stuck playback never finishes, so the "end" the stuck session sent would never have been acted on and the call would never have been connected again. - The stuck session now keeps what to say again (ConversationRelayResumes: in Redis when there is one, as the callback may run on another instance; on this instance otherwise, for 5 minutes) and drops its socket. Twilio ends the session at once and its callback finds it failed, as when an instance holding a socket goes away, which dev already recovers from (calls a2f3dffa, 469eeb86). - The action callback takes what was kept for the conversation, once, whatever status Twilio gives the dropped session, and connects the call again with it. Not for a session that ended with a handoff (a hang-up or a transfer), nor past MaxReconnects. - The "reconnect" handoff and its handoffData are gone. - A negative StuckPlaybackSeconds takes a playback still going that long before its expected end as stuck, so the reconnect can be tried on a test call: no real stuck playback can be made on demand. Tests (onebrain UnitTest/TwilioHooks/ConversationRelayWatchdogTests.StuckPlayback.cs): the stuck session drops the socket, keeps the unheard lines and sends no "end"; the callback connects the call again with what was kept, once; a hang-up or a call past its reconnects isn't connected again; a negative setting fires while Twilio still plays. 107 of 107 relay tests pass. Co-Authored-By: Claude Opus 5.5 --- .../TwilioConversationRelayController.cs | 47 +++-------- .../Models/Relay/ConversationRelayHandoff.cs | 11 --- .../Models/Relay/ConversationRelayResume.cs | 4 +- .../Services/ConversationRelayResumes.cs | 82 +++++++++++++++++++ .../Services/ConversationRelaySession.cs | 54 ++++++++---- .../Settings/ConversationRelaySetting.cs | 7 +- .../TwilioConversationRelayMiddleware.cs | 4 +- .../BotSharp.Plugin.Twilio/TwilioPlugin.cs | 1 + 8 files changed, 143 insertions(+), 67 deletions(-) create mode 100644 src/Plugins/BotSharp.Plugin.Twilio/Services/ConversationRelayResumes.cs diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioConversationRelayController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioConversationRelayController.cs index 1111bf937..ad58cebad 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioConversationRelayController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioConversationRelayController.cs @@ -12,17 +12,20 @@ public class TwilioConversationRelayController : TwilioController private readonly TwilioSetting _settings; private readonly IServiceProvider _services; private readonly ConversationRelaySessions _sessions; + private readonly ConversationRelayResumes _resumes; private readonly ILogger _logger; public TwilioConversationRelayController( TwilioSetting settings, IServiceProvider services, ConversationRelaySessions sessions, + ConversationRelayResumes resumes, ILogger logger) { _settings = settings; _services = services; _sessions = sessions; + _resumes = resumes; _logger = logger; } @@ -53,20 +56,15 @@ await HookEmitter.Emit(_services, var callStillUp = !string.Equals(request.CallStatus, "completed", StringComparison.OrdinalIgnoreCase); var canReconnect = request.Reconnects < _settings.ConversationRelay.MaxReconnects; - // The session's playback was stuck: a new session says what never finished playing. Nothing is told to the hooks: - // the call goes on. - if (action == ConversationRelayHandoff.Reconnect) + // A session whose playback was stuck dropped its socket and kept what never finished playing: a new session says it. + // Nothing is told to the hooks, as the call goes on. Whatever status Twilio gives the dropped session. + var resume = action == null && callStillUp && canReconnect ? await _resumes.TakeAsync(request.ConversationId) : null; + if (resume != null) { - if (callStillUp && canReconnect) - { - _logger.LogWarning($"Reconnecting ConversationRelay for conversation {request.ConversationId}: its playback " + - $"was stuck (attempt {request.Reconnects + 1})."); - return TwiML(twilio.ReturnConversationRelay(request.AgentId, request.ConversationId, welcomeGreeting: null, - request.Reconnects + 1, GetHandoffField(request.HandoffData, ConversationRelayHandoff.ResumeField))); - } - // The session asks only while it may (it knows its reconnects): a caller who hung up meanwhile ends the call - // as any hang-up does, and anything else as the agent's - action = callStillUp ? ConversationRelayHandoff.HangUp : null; + _logger.LogWarning($"Reconnecting ConversationRelay for conversation {request.ConversationId}: its playback was " + + $"stuck (attempt {request.Reconnects + 1})."); + return TwiML(twilio.ReturnConversationRelay(request.AgentId, request.ConversationId, welcomeGreeting: null, + request.Reconnects + 1, resume)); } if (request.SessionStatus == "failed" && callStillUp && canReconnect) @@ -120,27 +118,4 @@ await HookEmitter.Emit(_services, return handoffData; } } - - /// - /// A field of the handoffData as JSON, or null when it has none - /// - private static string? GetHandoffField(string? handoffData, string name) - { - if (string.IsNullOrWhiteSpace(handoffData)) - { - return null; - } - - try - { - using var doc = JsonDocument.Parse(handoffData); - return doc.RootElement.ValueKind == JsonValueKind.Object && doc.RootElement.TryGetProperty(name, out var field) - ? field.GetRawText() - : null; - } - catch (JsonException) - { - return null; - } - } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/Relay/ConversationRelayHandoff.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/Relay/ConversationRelayHandoff.cs index 005a80b76..1338b10df 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Models/Relay/ConversationRelayHandoff.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/Relay/ConversationRelayHandoff.cs @@ -7,15 +7,4 @@ public static class ConversationRelayHandoff { public const string HangUp = "hangup"; public const string Transfer = "transfer"; - - /// - /// The session's playback was stuck: the call is connected to a new session, which says what never finished playing - /// (ConversationRelaySetting.StuckPlaybackSeconds). The handoffData carries it under . - /// - public const string Reconnect = "reconnect"; - - /// - /// The handoffData field a carries its in - /// - public const string ResumeField = "resume"; } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/Relay/ConversationRelayResume.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/Relay/ConversationRelayResume.cs index 3c1c29005..f362ffc5b 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Models/Relay/ConversationRelayResume.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/Relay/ConversationRelayResume.cs @@ -5,8 +5,8 @@ namespace BotSharp.Plugin.Twilio.Models.Relay; /// all of, and how the flow last said to treat a silent caller /// /// -/// It goes from the stuck session to the new one through Twilio (the "end" message's handoffData, then a TwiML Parameter), -/// so the new session may run on another instance. +/// The stuck session keeps it (ConversationRelayResumes), Twilio's callback takes it into the new relay's TwiML as a +/// Parameter, and the new session reads it from its setup, so each may run on another instance. /// public class ConversationRelayResume { diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/ConversationRelayResumes.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/ConversationRelayResumes.cs new file mode 100644 index 000000000..6b1e805eb --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/ConversationRelayResumes.cs @@ -0,0 +1,82 @@ +using StackExchange.Redis; +using System.Collections.Concurrent; +using Task = System.Threading.Tasks.Task; + +namespace BotSharp.Plugin.Twilio.Services; + +/// +/// What a session connected again in place of one whose playback was stuck starts with +/// (), from the stuck session to Twilio's callback asking what next +/// +/// +/// Why kept here and not sent with the session's end: Twilio acts on an "end" message only once what is queued has played, +/// and a stuck playback never has, so the stuck session drops its socket instead, which carries nothing. The callback may +/// run on another instance: kept in Redis when there is one, in this instance otherwise. +/// +public class ConversationRelayResumes +{ + // Long enough for Twilio's callback, which comes within seconds of the socket dropping + private static readonly TimeSpan Kept = TimeSpan.FromMinutes(5); + + private readonly IServiceProvider _services; + private readonly ILogger _logger; + private readonly ConcurrentDictionary _here = new(); + + public ConversationRelayResumes(IServiceProvider services, ILogger logger) + { + _services = services; + _logger = logger; + } + + /// + /// Keeps what the session taking the place of 's stuck one starts with + /// + public async Task KeepAsync(string conversationId, string json) + { + if (_services.GetService() is { } redis) + { + try + { + await redis.GetDatabase().StringSetAsync(Key(conversationId), json, Kept); + return; + } + catch (RedisException ex) + { + _logger.LogWarning(ex, $"Could not keep the resume of conversation {conversationId} in Redis; kept on this instance."); + } + } + _here[conversationId] = (json, DateTime.UtcNow + Kept); + } + + /// + /// Takes what was kept for , once: null when nothing was, or it is gone + /// + public async Task TakeAsync(string conversationId) + { + if (string.IsNullOrEmpty(conversationId)) + { + return null; + } + + if (_services.GetService() is { } redis) + { + try + { + var db = redis.GetDatabase(); + var kept = await db.StringGetAsync(Key(conversationId)); + if (kept.HasValue) + { + await db.KeyDeleteAsync(Key(conversationId)); + return kept.ToString(); + } + } + catch (RedisException ex) + { + _logger.LogWarning(ex, $"Could not read the resume of conversation {conversationId} from Redis."); + } + } + return _here.TryRemove(conversationId, out var here) && here.UntilUtc > DateTime.UtcNow ? here.Json : null; + } + + private static string Key(string conversationId) => $"twilio:relay:resume:{conversationId}"; +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/ConversationRelaySession.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/ConversationRelaySession.cs index 5405e9006..266d02e87 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/ConversationRelaySession.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/ConversationRelaySession.cs @@ -171,6 +171,10 @@ public class ConversationRelaySession /// apology of a failed turn, an idle prompt and its goodbye, a line said again after a cut-in, and the fillers of a /// turn that kept none. A turn keeps its own reply, and a tool-call indication is the flow's to keep. /// + /// + /// Keeps what the session taking this one's place starts with (a as JSON) when the + /// playback is stuck, for Twilio's callback to hand on () + /// public ConversationRelaySession( WebSocket socket, string agentId, @@ -178,7 +182,8 @@ public ConversationRelaySession( TwilioSetting settings, Func> sendCallerMessage, ILogger logger, - Func? keepSaidLine = null) + Func? keepSaidLine = null, + Func? keepResume = null) { _socket = socket; _agentId = agentId; @@ -188,8 +193,11 @@ public ConversationRelaySession( _sendCallerMessage = sendCallerMessage; _logger = logger; _keepSaidLine = keepSaidLine; + _keepResume = keepResume; } + private readonly Func? _keepResume; + public async Task RunAsync(CancellationToken cancellationToken) { using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); @@ -198,7 +206,8 @@ public async Task RunAsync(CancellationToken cancellationToken) { await ReceiveLoopAsync(cts.Token); } - catch (Exception ex) when (ex is WebSocketException or OperationCanceledException) + // ObjectDisposedException: the socket dropped on purpose under a pending receive (ReconnectIfStuckAsync) + catch (Exception ex) when (ex is WebSocketException or OperationCanceledException or ObjectDisposedException) { _logger.LogInformation($"ConversationRelay socket closed for conversation {_conversationId}: {ex.Message}"); } @@ -321,7 +330,7 @@ private async Task HandleMessageAsync(string json) _playedByUtc = now; // Watching for a stuck playback, the caller's silence starts no sooner than the playback ended // (the estimate may be short of it): no idle prompt was said while it played (WatchIdleAsync) - if (_relay.StuckPlaybackSeconds > 0 && _quietSinceUtc != DateTime.MaxValue && _quietSinceUtc < now) + if (_relay.StuckPlaybackSeconds != 0 && _quietSinceUtc != DateTime.MaxValue && _quietSinceUtc < now) { _quietSinceUtc = now; } @@ -941,7 +950,7 @@ private async Task WatchIdleAsync(CancellationToken cancellationToken) // Watching for a stuck playback, a caller Twilio is still playing to isn't silent: an idle prompt then would // only queue behind the playback, and each one would put off finding it stuck until the idle prompts ran out // and hung up (ReconnectIfStuckAsync) - var stillPlaying = _relay.StuckPlaybackSeconds > 0 && _agentSpeaking; + var stillPlaying = _relay.StuckPlaybackSeconds != 0 && _agentSpeaking; if (_ended || _isProcessing || _quietSinceUtc == DateTime.MaxValue || _resumeLine != null || stillPlaying || quietFor < IdleTimeout(_idlePrompts, _quietWait)) { @@ -1029,6 +1038,10 @@ await SayOwnLineAsync(PromptText($"say-it-again-{Random.Shared.Next(1, 5)}", Def /// connected again (MaxReconnects): past that the action callback would hang it up. /// Why say the lines again rather than carry on: a stuck playback plays nothing queued behind it and lets none of the /// caller's words through, so the caller heard neither the lines nor anything after them (dev calls 1c28cefa, 46bc439c). + /// Why the socket is dropped rather than the session ended: Twilio acts on an "end" message only once what is queued has + /// played (dev: the session ended 5.4-6.0 s after a goodbye and its "end" went out together, the goodbye's length), and a + /// stuck playback never has. A dropped socket ends it at once: Twilio's callback then finds the session failed, as when + /// an instance holding a socket goes away, and connects the call again with what was kept (). /// private async Task ReconnectIfStuckAsync() { @@ -1037,7 +1050,7 @@ private async Task ReconnectIfStuckAsync() lock (_sync) { var stuckAfter = _relay.StuckPlaybackSeconds; - if (stuckAfter <= 0 || _ended || _isProcessing || !_agentSpeaking || _reconnects >= _relay.MaxReconnects) + if (stuckAfter == 0 || _ended || _isProcessing || !_agentSpeaking || _reconnects >= _relay.MaxReconnects) { return false; } @@ -1063,17 +1076,30 @@ private async Task ReconnectIfStuckAsync() } _logger.LogWarning($"ConversationRelay playback is stuck in conversation {_conversationId}: Twilio still says the " + - $"agent is speaking {overdueSeconds:0.0} s after all of it should have played. Connecting the call again to say " + - $"{resume.Lines.Count} line(s) again."); - await SendAsync(new + $"agent is speaking {overdueSeconds:0.0} s after all of it should have played. Dropping the socket so the call is " + + $"connected again, to say {resume.Lines.Count} line(s) again."); + if (_keepResume != null) { - type = "end", - handoffData = JsonSerializer.Serialize(new Dictionary + try { - ["action"] = ConversationRelayHandoff.Reconnect, - [ConversationRelayHandoff.ResumeField] = resume - }) - }); + await _keepResume(JsonSerializer.Serialize(resume)); + } + catch (Exception ex) + { + // Connected again all the same: a new session that says nothing beats a stuck one, and its idle prompts ask + _logger.LogWarning(ex, $"Could not keep what to say again in conversation {_conversationId}."); + } + } + // Under the send lock, so no frame is going out as the socket drops + await _sendLock.WaitAsync(); + try + { + _socket.Abort(); + } + finally + { + _sendLock.Release(); + } return true; } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Settings/ConversationRelaySetting.cs b/src/Plugins/BotSharp.Plugin.Twilio/Settings/ConversationRelaySetting.cs index 9cb143d0c..53126f73f 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Settings/ConversationRelaySetting.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Settings/ConversationRelaySetting.cs @@ -146,9 +146,10 @@ public class ConversationRelaySetting /// /// Seconds past the moment everything said should have finished playing, with Twilio still saying the agent is speaking - /// and nothing cut into, before the session's playback is taken as stuck: the session is ended, Twilio connects the - /// call again, and the lines that never finished playing are said on the new session. Needs Twilio's speaker events - /// (the events attribute with speaker-events); stops after . 0 turns it off. + /// and nothing cut into, before the session's playback is taken as stuck: the session drops its socket, Twilio connects + /// the call again, and the lines that never finished playing are said on the new session. Needs Twilio's speaker events + /// (the events attribute with speaker-events); stops after . 0 turns it off. A negative value + /// takes a playback still going that long before its expected end as stuck: only to try the reconnect on a test call. /// /// /// Why: on some calls Twilio said the agent started speaking and never that it stopped. Nothing said after that played, diff --git a/src/Plugins/BotSharp.Plugin.Twilio/TwilioConversationRelayMiddleware.cs b/src/Plugins/BotSharp.Plugin.Twilio/TwilioConversationRelayMiddleware.cs index baf0442a0..bda7c2b98 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/TwilioConversationRelayMiddleware.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/TwilioConversationRelayMiddleware.cs @@ -47,10 +47,12 @@ public async Task Invoke(HttpContext httpContext) using var socket = await httpContext.WebSockets.AcceptWebSocketAsync(); var scopeFactory = services.GetRequiredService(); + var resumes = services.GetRequiredService(); var session = new ConversationRelaySession(socket, agentId, conversationId, settings, (message, callbacks) => SendCallerMessageAsync(scopeFactory, message, callbacks), _logger, - (line, label, saidUtc) => KeepSaidLineAsync(scopeFactory, agentId, conversationId, line, label, saidUtc)); + (line, label, saidUtc) => KeepSaidLineAsync(scopeFactory, agentId, conversationId, line, label, saidUtc), + resume => resumes.KeepAsync(conversationId, resume)); await services.GetRequiredService() .RunAsync(conversationId, () => session.RunAsync(httpContext.RequestAborted)); } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs b/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs index cd55abec9..3e96faa05 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs @@ -31,6 +31,7 @@ public void RegisterDI(IServiceCollection services, IConfiguration config) services.AddSingleton(sessionManager); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); services.AddHostedService(); services.AddTwilioRequestValidation(); services.AddScoped(); From 22c1f3976cf81b146b89c209f27b977516201333 Mon Sep 17 00:00:00 2001 From: "mars.yu" <18671933+yuyixg@users.noreply.github.com> Date: Wed, 18 Nov 2026 16:15:19 +0800 Subject: [PATCH 3/3] fix(twilio): say again every line a stuck playback held, and ask Twilio for the speaker events it is found by Review of #1500: - The lines to say again are kept apart from the last eight utterances kept for cut-ins, all of them since Twilio last said the playback stopped: a stuck playback plays none of what went out after it, however much that is. - No more falling back to the last line when nothing to say again went out since then: after an idle prompt or a filler that line had already played. The case it was for, a line that crossed Twilio's "agentSpeaking off", is now kept as one: a line that went out less than 1 s before that message is not taken as played, and is waited for before the playback counts as stuck. - What was kept for the new session is read and deleted in one Redis transaction, so two callbacks can't both take it (not GETDEL, which needs Redis 6.2). - A relay watched for a stuck playback asks for Twilio's speaker events itself (events="speaker-events") instead of relying on the host adding the attribute. Tests (onebrain UnitTest.TwilioHooks, built against this branch): every line kept past eight, a line that crossed the "off" is said again and waited for, a stuck idle prompt says nothing again, the events attribute. Co-Authored-By: Claude Opus 5.5 --- .../Services/ConversationRelayResumes.cs | 15 +++-- .../Services/ConversationRelaySession.cs | 58 ++++++++++++------- .../Services/TwilioService.cs | 5 ++ .../Settings/ConversationRelaySetting.cs | 4 +- 4 files changed, 53 insertions(+), 29 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/ConversationRelayResumes.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/ConversationRelayResumes.cs index 6b1e805eb..a0c16733c 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/ConversationRelayResumes.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/ConversationRelayResumes.cs @@ -62,12 +62,17 @@ public async Task KeepAsync(string conversationId, string json) { try { - var db = redis.GetDatabase(); - var kept = await db.StringGetAsync(Key(conversationId)); - if (kept.HasValue) + // Read and deleted in one transaction, so two callbacks at once (Twilio retrying one) can't both take it. + // Not GETDEL, which needs Redis 6.2. + var transaction = redis.GetDatabase().CreateTransaction(); + var kept = transaction.StringGetAsync(Key(conversationId)); + var deleted = transaction.KeyDeleteAsync(Key(conversationId)); + await transaction.ExecuteAsync(); + await deleted; + var value = await kept; + if (value.HasValue) { - await db.KeyDeleteAsync(Key(conversationId)); - return kept.ToString(); + return value.ToString(); } } catch (RedisException ex) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/ConversationRelaySession.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/ConversationRelaySession.cs index 266d02e87..f20b1d26a 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/ConversationRelaySession.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/ConversationRelaySession.cs @@ -134,16 +134,22 @@ public class ConversationRelaySession private bool _lastLineHeardSome; // What was said, newest last, with _saidCount as it went out: tells which utterance Twilio says was playing at a - // cut-in, and which never finished playing when the playback got stuck (with whether it went out to play to the end, - // and whether it is one to say again then: not a filler or an idle prompt). Only the last few are kept. - private readonly List<(string Text, int Mark, bool KeptUninterrupted, bool Again)> _utterances = []; + // cut-in. Only the last few are kept. + private readonly List<(string Text, int Mark)> _utterances = []; private const int UtterancesKept = 8; - // Twilio's speaker events (the events attribute with speaker-events): whether it last said the agent is speaking, and - // _saidCount when it last said the playback stopped (all of it played, or the caller cut in). What was said after that - // isn't known to have been heard. + // Twilio's speaker events (the events attribute with speaker-events): whether it last said the agent is speaking private bool _agentSpeaking; - private int _playedThroughMark; + // The lines said since Twilio last said the playback stopped (all of it played, or the caller cut in), with whether + // each went out to play to the end and when: the ones to say again on a new session if the playback gets stuck + // (ReconnectIfStuckAsync). Not fillers or idle prompts, which were about a moment that has passed. Kept only while + // watching for a stuck playback, and all of them, unlike _utterances: a stuck playback plays none of what went out + // after it, however much that is. + private readonly List<(string Text, bool KeptUninterrupted, DateTime SentUtc)> _unheard = []; + // A line that went out this soon before Twilio said the playback stopped may have crossed that message: Twilio + // stopped before the line came, and plays it next. A line it played in that playback went out at least as long before + // as it takes to play, which is longer for nearly any line; one that isn't is only said again needlessly. + private static readonly TimeSpan CrossedPlaybackStop = TimeSpan.FromSeconds(1); // When everything said should have finished playing, by its length (EstimateSpeech) private DateTime _playedByUtc = DateTime.MinValue; // How many times the call's relay has been connected again (ConversationRelayParameters.Reconnects) @@ -323,11 +329,8 @@ private async Task HandleMessageAsync(string json) _agentSpeaking = GetString(root, "value") == "on"; if (!_agentSpeaking) { - _playedThroughMark = _saidCount; - // Nothing is left to play: what goes out next plays from now, not after what the estimate - // still had to come var now = DateTime.UtcNow; - _playedByUtc = now; + NotePlaybackStopped(now); // Watching for a stuck playback, the caller's silence starts no sooner than the playback ended // (the estimate may be short of it): no idle prompt was said while it played (WatchIdleAsync) if (_relay.StuckPlaybackSeconds != 0 && _quietSinceUtc != DateTime.MaxValue && _quietSinceUtc < now) @@ -398,8 +401,7 @@ private async Task HandleMessageAsync(string json) // The playback stopped where the caller cut in: what is said again after a cut-in is the cut-in's to // decide (ResumeAfterCutInSeconds), not a stuck playback's _agentSpeaking = false; - _playedThroughMark = _saidCount; - _playedByUtc = DateTime.UtcNow; + NotePlaybackStopped(DateTime.UtcNow); // The caller talked over a reply still being streamed: don't send the rest of it. A stream already // dropped (the turn timed out and the apology is playing, or the caller cut in before) or a @@ -881,11 +883,27 @@ private void RememberUtterance(string text, int mark, bool keptUninterrupted = f { return; } - _utterances.Add((text.Trim(), mark, keptUninterrupted, again)); + _utterances.Add((text.Trim(), mark)); if (_utterances.Count > UtterancesKept) { _utterances.RemoveAt(0); } + if (again && _relay.StuckPlaybackSeconds != 0) + { + _unheard.Add((text.Trim(), keptUninterrupted, DateTime.UtcNow)); + } + } + + /// + /// Twilio said the playback stopped: all of it played, or the caller cut in. Call under . + /// + private void NotePlaybackStopped(DateTime now) + { + // What went out before it is over, but for a line that may have crossed Twilio's message (CrossedPlaybackStop) + _unheard.RemoveAll(line => line.SentUtc < now - CrossedPlaybackStop); + // Nothing is left to play but such a line: what goes out next plays after it, not after what the estimate still + // had to come. Without it, a long line that crossed the message would be taken as stuck while it still plays. + _playedByUtc = now + _unheard.Aggregate(TimeSpan.Zero, (sum, line) => sum + EstimateSpeech(line.Text.Length)); } /// @@ -1060,15 +1078,11 @@ private async Task ReconnectIfStuckAsync() return false; } - var lines = _utterances.Where(u => u.Mark > _playedThroughMark && u.Again) - .Select(u => new ConversationRelayResumeLine { Text = u.Text, KeepUninterrupted = u.KeptUninterrupted }) + // None when only a filler or an idle prompt went out since the playback last stopped: the new session then says + // nothing until the caller does, or its own idle prompt + var lines = _unheard + .Select(line => new ConversationRelayResumeLine { Text = line.Text, KeepUninterrupted = line.KeptUninterrupted }) .ToList(); - // Nothing to say again went out since the playback last stopped (it went out just as Twilio said so): the last - // line, which the caller may not have heard either - if (lines.Count == 0 && _lastLine != null) - { - lines.Add(new ConversationRelayResumeLine { Text = _lastLine, KeepUninterrupted = _lastLineKeptUninterrupted }); - } resume = new ConversationRelayResume { Lines = lines, QuietWait = _quietWait, GoodbyeOnSilence = _goodbyeOnSilence }; overdueSeconds = overdue.TotalSeconds; // Ended in the same step, so caller input arriving now can't start a turn on a session being replaced diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs index 7ec986324..4353bdc5c 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs @@ -308,6 +308,11 @@ public VoiceResponse ReturnConversationRelay( { element.SetOption("hints", relay.Hints); } + // Watching for a stuck playback reads when Twilio says the agent starts and stops speaking + if (relay.StuckPlaybackSeconds != 0) + { + element.SetOption("events", "speaker-events"); + } // Lets the session time its idle check from the end of the greeting var greetingLength = (welcomeGreeting?.Length ?? 0).ToString(); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Settings/ConversationRelaySetting.cs b/src/Plugins/BotSharp.Plugin.Twilio/Settings/ConversationRelaySetting.cs index 53126f73f..9c3e533e6 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Settings/ConversationRelaySetting.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Settings/ConversationRelaySetting.cs @@ -147,8 +147,8 @@ public class ConversationRelaySetting /// /// Seconds past the moment everything said should have finished playing, with Twilio still saying the agent is speaking /// and nothing cut into, before the session's playback is taken as stuck: the session drops its socket, Twilio connects - /// the call again, and the lines that never finished playing are said on the new session. Needs Twilio's speaker events - /// (the events attribute with speaker-events); stops after . 0 turns it off. A negative value + /// the call again, and the lines that never finished playing are said on the new session. Asks Twilio for its speaker + /// events (the events attribute with speaker-events); stops after . 0 turns it off. A negative value /// takes a playback still going that long before its expected end as stuck: only to try the reconnect on a test call. /// ///