diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioConversationRelayController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioConversationRelayController.cs index 294b37077..ad58cebad 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioConversationRelayController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioConversationRelayController.cs @@ -12,17 +12,20 @@ public class TwilioConversationRelayController : TwilioController private readonly TwilioSetting _settings; private readonly IServiceProvider _services; private readonly ConversationRelaySessions _sessions; + private readonly ConversationRelayResumes _resumes; private readonly ILogger _logger; public TwilioConversationRelayController( TwilioSetting settings, IServiceProvider services, ConversationRelaySessions sessions, + ConversationRelayResumes resumes, ILogger logger) { _settings = settings; _services = services; _sessions = sessions; + _resumes = resumes; _logger = logger; } @@ -52,6 +55,18 @@ await HookEmitter.Emit(_services, var callStillUp = !string.Equals(request.CallStatus, "completed", StringComparison.OrdinalIgnoreCase); var canReconnect = request.Reconnects < _settings.ConversationRelay.MaxReconnects; + + // A session whose playback was stuck dropped its socket and kept what never finished playing: a new session says it. + // Nothing is told to the hooks, as the call goes on. Whatever status Twilio gives the dropped session. + var resume = action == null && callStillUp && canReconnect ? await _resumes.TakeAsync(request.ConversationId) : null; + if (resume != null) + { + _logger.LogWarning($"Reconnecting ConversationRelay for conversation {request.ConversationId}: its playback was " + + $"stuck (attempt {request.Reconnects + 1})."); + return TwiML(twilio.ReturnConversationRelay(request.AgentId, request.ConversationId, welcomeGreeting: null, + request.Reconnects + 1, resume)); + } + if (request.SessionStatus == "failed" && callStillUp && canReconnect) { _logger.LogWarning($"Reconnecting ConversationRelay for conversation {request.ConversationId} " + diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/Relay/ConversationRelayParameters.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/Relay/ConversationRelayParameters.cs index b8b78f40c..294b0c531 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Models/Relay/ConversationRelayParameters.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/Relay/ConversationRelayParameters.cs @@ -6,4 +6,14 @@ namespace BotSharp.Plugin.Twilio.Models.Relay; public static class ConversationRelayParameters { public const string WelcomeGreetingLength = "welcome-greeting-length"; + + /// + /// How many times the call's relay has been connected again + /// + public const string Reconnects = "reconnects"; + + /// + /// What a session connected again in place of one whose playback was stuck starts with () + /// + public const string Resume = "resume"; } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/Relay/ConversationRelayResume.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/Relay/ConversationRelayResume.cs new file mode 100644 index 000000000..f362ffc5b --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/Relay/ConversationRelayResume.cs @@ -0,0 +1,37 @@ +namespace BotSharp.Plugin.Twilio.Models.Relay; + +/// +/// What a session connected again in place of one whose playback was stuck starts with: the lines the caller never heard +/// all of, and how the flow last said to treat a silent caller +/// +/// +/// The stuck session keeps it (ConversationRelayResumes), Twilio's callback takes it into the new relay's TwiML as a +/// Parameter, and the new session reads it from its setup, so each may run on another instance. +/// +public class ConversationRelayResume +{ + /// + /// The lines said since Twilio last said playback stopped, in order, but fillers and idle prompts + /// + public List Lines { get; set; } = []; + + /// + /// The flow was waiting quietly for the caller (ConversationRelayStates.QuietWait) + /// + public bool QuietWait { get; set; } + + /// + /// What the flow said to say if the caller stays silent until the hang-up (ConversationRelayStates.GoodbyeOnSilence) + /// + public string? GoodbyeOnSilence { get; set; } +} + +public class ConversationRelayResumeLine +{ + public string Text { get; set; } = string.Empty; + + /// + /// It went out to play to the end whoever speaks over it, and goes out so again + /// + public bool KeepUninterrupted { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/ConversationRelayResumes.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/ConversationRelayResumes.cs new file mode 100644 index 000000000..a0c16733c --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/ConversationRelayResumes.cs @@ -0,0 +1,87 @@ +using StackExchange.Redis; +using System.Collections.Concurrent; +using Task = System.Threading.Tasks.Task; + +namespace BotSharp.Plugin.Twilio.Services; + +/// +/// What a session connected again in place of one whose playback was stuck starts with +/// (), from the stuck session to Twilio's callback asking what next +/// +/// +/// Why kept here and not sent with the session's end: Twilio acts on an "end" message only once what is queued has played, +/// and a stuck playback never has, so the stuck session drops its socket instead, which carries nothing. The callback may +/// run on another instance: kept in Redis when there is one, in this instance otherwise. +/// +public class ConversationRelayResumes +{ + // Long enough for Twilio's callback, which comes within seconds of the socket dropping + private static readonly TimeSpan Kept = TimeSpan.FromMinutes(5); + + private readonly IServiceProvider _services; + private readonly ILogger _logger; + private readonly ConcurrentDictionary _here = new(); + + public ConversationRelayResumes(IServiceProvider services, ILogger logger) + { + _services = services; + _logger = logger; + } + + /// + /// Keeps what the session taking the place of 's stuck one starts with + /// + public async Task KeepAsync(string conversationId, string json) + { + if (_services.GetService() is { } redis) + { + try + { + await redis.GetDatabase().StringSetAsync(Key(conversationId), json, Kept); + return; + } + catch (RedisException ex) + { + _logger.LogWarning(ex, $"Could not keep the resume of conversation {conversationId} in Redis; kept on this instance."); + } + } + _here[conversationId] = (json, DateTime.UtcNow + Kept); + } + + /// + /// Takes what was kept for , once: null when nothing was, or it is gone + /// + public async Task TakeAsync(string conversationId) + { + if (string.IsNullOrEmpty(conversationId)) + { + return null; + } + + if (_services.GetService() is { } redis) + { + try + { + // Read and deleted in one transaction, so two callbacks at once (Twilio retrying one) can't both take it. + // Not GETDEL, which needs Redis 6.2. + var transaction = redis.GetDatabase().CreateTransaction(); + var kept = transaction.StringGetAsync(Key(conversationId)); + var deleted = transaction.KeyDeleteAsync(Key(conversationId)); + await transaction.ExecuteAsync(); + await deleted; + var value = await kept; + if (value.HasValue) + { + return value.ToString(); + } + } + catch (RedisException ex) + { + _logger.LogWarning(ex, $"Could not read the resume of conversation {conversationId} from Redis."); + } + } + return _here.TryRemove(conversationId, out var here) && here.UntilUtc > DateTime.UtcNow ? here.Json : null; + } + + private static string Key(string conversationId) => $"twilio:relay:resume:{conversationId}"; +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/ConversationRelaySession.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/ConversationRelaySession.cs index 1bac875b5..f20b1d26a 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/ConversationRelaySession.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/ConversationRelaySession.cs @@ -138,6 +138,23 @@ public class ConversationRelaySession private readonly List<(string Text, int Mark)> _utterances = []; private const int UtterancesKept = 8; + // Twilio's speaker events (the events attribute with speaker-events): whether it last said the agent is speaking + private bool _agentSpeaking; + // The lines said since Twilio last said the playback stopped (all of it played, or the caller cut in), with whether + // each went out to play to the end and when: the ones to say again on a new session if the playback gets stuck + // (ReconnectIfStuckAsync). Not fillers or idle prompts, which were about a moment that has passed. Kept only while + // watching for a stuck playback, and all of them, unlike _utterances: a stuck playback plays none of what went out + // after it, however much that is. + private readonly List<(string Text, bool KeptUninterrupted, DateTime SentUtc)> _unheard = []; + // A line that went out this soon before Twilio said the playback stopped may have crossed that message: Twilio + // stopped before the line came, and plays it next. A line it played in that playback went out at least as long before + // as it takes to play, which is longer for nearly any line; one that isn't is only said again needlessly. + private static readonly TimeSpan CrossedPlaybackStop = TimeSpan.FromSeconds(1); + // When everything said should have finished playing, by its length (EstimateSpeech) + private DateTime _playedByUtc = DateTime.MinValue; + // How many times the call's relay has been connected again (ConversationRelayParameters.Reconnects) + private int _reconnects; + // The line the caller cut into, to say again once nothing follows the cut-in (ResumeAfterCutInSeconds) private string? _resumeLine; private DateTime _cutInUtc; @@ -160,6 +177,10 @@ public class ConversationRelaySession /// apology of a failed turn, an idle prompt and its goodbye, a line said again after a cut-in, and the fillers of a /// turn that kept none. A turn keeps its own reply, and a tool-call indication is the flow's to keep. /// + /// + /// Keeps what the session taking this one's place starts with (a as JSON) when the + /// playback is stuck, for Twilio's callback to hand on () + /// public ConversationRelaySession( WebSocket socket, string agentId, @@ -167,7 +188,8 @@ public ConversationRelaySession( TwilioSetting settings, Func> sendCallerMessage, ILogger logger, - Func? keepSaidLine = null) + Func? keepSaidLine = null, + Func? keepResume = null) { _socket = socket; _agentId = agentId; @@ -177,8 +199,11 @@ public ConversationRelaySession( _sendCallerMessage = sendCallerMessage; _logger = logger; _keepSaidLine = keepSaidLine; + _keepResume = keepResume; } + private readonly Func? _keepResume; + public async Task RunAsync(CancellationToken cancellationToken) { using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); @@ -187,7 +212,8 @@ public async Task RunAsync(CancellationToken cancellationToken) { await ReceiveLoopAsync(cts.Token); } - catch (Exception ex) when (ex is WebSocketException or OperationCanceledException) + // ObjectDisposedException: the socket dropped on purpose under a pending receive (ReconnectIfStuckAsync) + catch (Exception ex) when (ex is WebSocketException or OperationCanceledException or ObjectDisposedException) { _logger.LogInformation($"ConversationRelay socket closed for conversation {_conversationId}: {ex.Message}"); } @@ -265,17 +291,55 @@ private async Task HandleMessageAsync(string json) _to = GetString(root, "to"); _direction = GetString(root, "direction"); var greetingLength = 0; - if (root.TryGetProperty("customParameters", out var parameters) - && GetString(parameters, ConversationRelayParameters.WelcomeGreetingLength) is string length) + var reconnects = 0; + string? resume = null; + if (root.TryGetProperty("customParameters", out var parameters)) { - int.TryParse(length, out greetingLength); + if (GetString(parameters, ConversationRelayParameters.WelcomeGreetingLength) is string length) + { + int.TryParse(length, out greetingLength); + } + if (GetString(parameters, ConversationRelayParameters.Reconnects) is string count) + { + int.TryParse(count, out reconnects); + } + resume = GetString(parameters, ConversationRelayParameters.Resume); } lock (_sync) { _quietSinceUtc = DateTime.UtcNow + EstimateSpeech(greetingLength); + _playedByUtc = _quietSinceUtc; + _reconnects = reconnects; } _logger.LogInformation( $"ConversationRelay session started for conversation {_conversationId} (call {_callSid})."); + if (resume != null) + { + await ResumeAsync(resume); + } + break; + + case "info": + // Twilio's speaker events. One "on" and one "off" cover everything played back to back, however many lines + // went out meanwhile: at "off" all of it has played. + if (GetString(root, "name") == "agentSpeaking") + { + lock (_sync) + { + _agentSpeaking = GetString(root, "value") == "on"; + if (!_agentSpeaking) + { + var now = DateTime.UtcNow; + NotePlaybackStopped(now); + // Watching for a stuck playback, the caller's silence starts no sooner than the playback ended + // (the estimate may be short of it): no idle prompt was said while it played (WatchIdleAsync) + if (_relay.StuckPlaybackSeconds != 0 && _quietSinceUtc != DateTime.MaxValue && _quietSinceUtc < now) + { + _quietSinceUtc = now; + } + } + } + } break; case "prompt": @@ -334,6 +398,10 @@ private async Task HandleMessageAsync(string json) lock (_sync) { _quietSinceUtc = DateTime.UtcNow; + // The playback stopped where the caller cut in: what is said again after a cut-in is the cut-in's to + // decide (ResumeAfterCutInSeconds), not a stuck playback's + _agentSpeaking = false; + NotePlaybackStopped(DateTime.UtcNow); // The caller talked over a reply still being streamed: don't send the rest of it. A stream already // dropped (the turn timed out and the apology is playing, or the caller cut in before) or a @@ -772,7 +840,7 @@ private async Task SpeakFillerAsync(int seqNumber, string text, TimeSpan i { return false; } - RememberUtterance(text, _saidCount + 1); + RememberUtterance(text, _saidCount + 1, again: false); // Noted as it goes out, under the send lock: a turn that fails meanwhile finds every filler said before // its apology callbacks.NoteFillerSaid(text); @@ -806,7 +874,10 @@ private void NoteLine(string? line, DateTime? sentUtc = null, bool keepHeard = f /// /// Notes something said, with the _saidCount it went out at. Call under . /// - private void RememberUtterance(string text, int mark) + /// It went out to play to the end whoever speaks over it + /// Said again on a new session if it never finished playing (ReconnectIfStuckAsync): not for a + /// filler or an idle prompt, which were about a moment that has passed + private void RememberUtterance(string text, int mark, bool keptUninterrupted = false, bool again = true) { if (string.IsNullOrWhiteSpace(text)) { @@ -817,6 +888,22 @@ private void RememberUtterance(string text, int mark) { _utterances.RemoveAt(0); } + if (again && _relay.StuckPlaybackSeconds != 0) + { + _unheard.Add((text.Trim(), keptUninterrupted, DateTime.UtcNow)); + } + } + + /// + /// Twilio said the playback stopped: all of it played, or the caller cut in. Call under . + /// + private void NotePlaybackStopped(DateTime now) + { + // What went out before it is over, but for a line that may have crossed Twilio's message (CrossedPlaybackStop) + _unheard.RemoveAll(line => line.SentUtc < now - CrossedPlaybackStop); + // Nothing is left to play but such a line: what goes out next plays after it, not after what the estimate still + // had to come. Without it, a long line that crossed the message would be taken as stuck while it still plays. + _playedByUtc = now + _unheard.Aggregate(TimeSpan.Zero, (sum, line) => sum + EstimateSpeech(line.Text.Length)); } /// @@ -862,6 +949,10 @@ private async Task WatchIdleAsync(CancellationToken cancellationToken) while (!cancellationToken.IsCancellationRequested) { await Task.Delay(TimeSpan.FromMilliseconds(500), cancellationToken); + if (await ReconnectIfStuckAsync()) + { + return; + } await ResumeIfSilentAsync(); var askAgain = false; @@ -874,7 +965,11 @@ private async Task WatchIdleAsync(CancellationToken cancellationToken) lock (_sync) { var quietFor = DateTime.UtcNow - _quietSinceUtc; - if (_ended || _isProcessing || _quietSinceUtc == DateTime.MaxValue || _resumeLine != null + // Watching for a stuck playback, a caller Twilio is still playing to isn't silent: an idle prompt then would + // only queue behind the playback, and each one would put off finding it stuck until the idle prompts ran out + // and hung up (ReconnectIfStuckAsync) + var stillPlaying = _relay.StuckPlaybackSeconds != 0 && _agentSpeaking; + if (_ended || _isProcessing || _quietSinceUtc == DateTime.MaxValue || _resumeLine != null || stillPlaying || quietFor < IdleTimeout(_idlePrompts, _quietWait)) { continue; @@ -951,6 +1046,120 @@ await SayOwnLineAsync(PromptText($"say-it-again-{Random.Shared.Next(1, 5)}", Def } } + /// + /// Ends a session whose playback is stuck (ConversationRelaySetting.StuckPlaybackSeconds) and has Twilio connect the call + /// again: the new session says the lines that never finished playing (). Whether it did. + /// + /// + /// Stuck: Twilio still says the agent is speaking well after everything said should have finished playing, and no turn + /// is running (one still running may say more, which then goes with the rest). Only while the call may still be + /// connected again (MaxReconnects): past that the action callback would hang it up. + /// Why say the lines again rather than carry on: a stuck playback plays nothing queued behind it and lets none of the + /// caller's words through, so the caller heard neither the lines nor anything after them (dev calls 1c28cefa, 46bc439c). + /// Why the socket is dropped rather than the session ended: Twilio acts on an "end" message only once what is queued has + /// played (dev: the session ended 5.4-6.0 s after a goodbye and its "end" went out together, the goodbye's length), and a + /// stuck playback never has. A dropped socket ends it at once: Twilio's callback then finds the session failed, as when + /// an instance holding a socket goes away, and connects the call again with what was kept (). + /// + private async Task ReconnectIfStuckAsync() + { + ConversationRelayResume resume; + double overdueSeconds; + lock (_sync) + { + var stuckAfter = _relay.StuckPlaybackSeconds; + if (stuckAfter == 0 || _ended || _isProcessing || !_agentSpeaking || _reconnects >= _relay.MaxReconnects) + { + return false; + } + var overdue = DateTime.UtcNow - _playedByUtc; + if (overdue < TimeSpan.FromSeconds(stuckAfter)) + { + return false; + } + + // None when only a filler or an idle prompt went out since the playback last stopped: the new session then says + // nothing until the caller does, or its own idle prompt + var lines = _unheard + .Select(line => new ConversationRelayResumeLine { Text = line.Text, KeepUninterrupted = line.KeptUninterrupted }) + .ToList(); + resume = new ConversationRelayResume { Lines = lines, QuietWait = _quietWait, GoodbyeOnSilence = _goodbyeOnSilence }; + overdueSeconds = overdue.TotalSeconds; + // Ended in the same step, so caller input arriving now can't start a turn on a session being replaced + _ended = true; + } + + _logger.LogWarning($"ConversationRelay playback is stuck in conversation {_conversationId}: Twilio still says the " + + $"agent is speaking {overdueSeconds:0.0} s after all of it should have played. Dropping the socket so the call is " + + $"connected again, to say {resume.Lines.Count} line(s) again."); + if (_keepResume != null) + { + try + { + await _keepResume(JsonSerializer.Serialize(resume)); + } + catch (Exception ex) + { + // Connected again all the same: a new session that says nothing beats a stuck one, and its idle prompts ask + _logger.LogWarning(ex, $"Could not keep what to say again in conversation {_conversationId}."); + } + } + // Under the send lock, so no frame is going out as the socket drops + await _sendLock.WaitAsync(); + try + { + _socket.Abort(); + } + finally + { + _sendLock.Release(); + } + return true; + } + + /// + /// Starts a session that takes the place of one whose playback was stuck (): the + /// lines the caller never heard all of are said again, each as it went out, and a silent caller is treated as the flow + /// last said. They are in the history already, so they aren't kept again. + /// + private async Task ResumeAsync(string json) + { + ConversationRelayResume? resume; + try + { + resume = JsonSerializer.Deserialize(json); + } + catch (JsonException ex) + { + _logger.LogWarning($"Unreadable resume for conversation {_conversationId}; nothing is said again: {ex.Message}"); + return; + } + if (resume == null) + { + return; + } + + lock (_sync) + { + _quietWait = resume.QuietWait; + _goodbyeOnSilence = resume.GoodbyeOnSilence; + } + var said = 0; + foreach (var line in resume.Lines) + { + if (await SpeakAsync(line.Text, canSpeak: () => + { + NoteLine(line.Text, keptUninterrupted: line.KeepUninterrupted); + return true; + }, keepUninterrupted: line.KeepUninterrupted)) + { + said++; + } + } + _logger.LogInformation($"ConversationRelay session for conversation {_conversationId} took the place of one whose " + + $"playback was stuck; said {said} line(s) again."); + } + // How long the caller may stay silent before the idle prompt after this many (the last is before the hang-up) private TimeSpan IdleTimeout(int prompts, bool quietWait) { @@ -1087,7 +1296,7 @@ private async Task KeepFillersLeftAsync(AgentTurnCallbacks callbacks) /// Extra condition, checked under once the send lock is held private async Task SayOwnLineAsync(string text, string label, Func? canSpeak = null) { - if (await SpeakAsync(text, canSpeak)) + if (await SpeakAsync(text, canSpeak, sayAgainIfStuck: label != ConversationRelayLabels.IdlePrompt)) { await KeepSaidAsync(text, label, DateTime.UtcNow); } @@ -1179,7 +1388,7 @@ await SendIfAsync(() => _lastStreamSegment = _streamSegment.ToString(); _streamSegment.Clear(); MarkSpeaking(EstimateSpeech(_lastStreamSegment.Length)); - RememberUtterance(_lastStreamSegment, _utteranceStartMark); + RememberUtterance(_lastStreamSegment, _utteranceStartMark, keep); // Cut into while it streamed: the caller's words are about it, so it dates from when it started and keeps // what they heard of it, not from now (a "Yes." said over it was dropped as a tail, review) if (_utteranceCutIn) @@ -1373,8 +1582,10 @@ await SpeakIfAsync(() => !_ended && (canSpeak?.Invoke() ?? true), /// Extra condition, checked under once the send lock is held /// The caller must hear all of it, so it plays to the end whoever speaks over it + /// Said again on a new session if it never finished playing (RememberUtterance) /// Whether it went out - private async Task SpeakAsync(string? text, Func? canSpeak = null, bool keepUninterrupted = false) + private async Task SpeakAsync(string? text, Func? canSpeak = null, bool keepUninterrupted = false, + bool sayAgainIfStuck = true) { if (string.IsNullOrWhiteSpace(text)) { @@ -1387,7 +1598,7 @@ private async Task SpeakAsync(string? text, Func? canSpeak = null, b { return false; } - RememberUtterance(text, _saidCount + 1); + RememberUtterance(text, _saidCount + 1, keepUninterrupted, sayAgainIfStuck); return true; }, TextFrame(text, last: true, keepUninterrupted), EstimateSpeech(text.Length)); @@ -1560,6 +1771,8 @@ private void MarkSpeaking(TimeSpan duration) var now = DateTime.UtcNow; var start = _quietSinceUtc == DateTime.MaxValue || _quietSinceUtc < now ? now : _quietSinceUtc; _quietSinceUtc = start + duration; + // Queued behind what is still to play; unlike the silence clock, the caller's words don't move it + _playedByUtc = (_playedByUtc < now ? now : _playedByUtc) + duration; } private static TimeSpan EstimateSpeech(int characters) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs index 6078d950f..4353bdc5c 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs @@ -272,8 +272,11 @@ public VoiceResponse ReturnBidirectionalMediaStreamsInstructions(ConversationalV /// Hands the call to Twilio ConversationRelay. Twilio opens a WebSocket to twilio/relay/{agentId}/{conversationId} /// and posts to twilio/relay/action when the session ends. /// + /// How many times the call's relay has been connected again + /// What the session starts with when it takes the place of one whose playback was stuck: a + /// as JSON public VoiceResponse ReturnConversationRelay( - string agentId, string conversationId, string? welcomeGreeting, int reconnects = 0) + string agentId, string conversationId, string? welcomeGreeting, int reconnects = 0, string? resume = null) { var relay = _settings.ConversationRelay.For(agentId); var query = $"agent-id={Uri.EscapeDataString(agentId)}" + @@ -305,10 +308,21 @@ public VoiceResponse ReturnConversationRelay( { element.SetOption("hints", relay.Hints); } + // Watching for a stuck playback reads when Twilio says the agent starts and stops speaking + if (relay.StuckPlaybackSeconds != 0) + { + element.SetOption("events", "speaker-events"); + } // Lets the session time its idle check from the end of the greeting var greetingLength = (welcomeGreeting?.Length ?? 0).ToString(); element.Parameter(name: ConversationRelayParameters.WelcomeGreetingLength, value: greetingLength); + // The session reconnects a stuck playback only while the call may still be connected again + element.Parameter(name: ConversationRelayParameters.Reconnects, value: reconnects.ToString()); + if (!string.IsNullOrWhiteSpace(resume)) + { + element.Parameter(name: ConversationRelayParameters.Resume, value: resume); + } connect.Append(element); var response = new VoiceResponse(); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Settings/ConversationRelayAgentSetting.cs b/src/Plugins/BotSharp.Plugin.Twilio/Settings/ConversationRelayAgentSetting.cs index 67a9823c5..7171a223e 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Settings/ConversationRelayAgentSetting.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Settings/ConversationRelayAgentSetting.cs @@ -23,4 +23,5 @@ public class ConversationRelayAgentSetting public int? MaxQuietWaitPrompts { get; set; } public double? ResumeAfterCutInSeconds { get; set; } public bool? DropBackchannelTails { get; set; } + public double? StuckPlaybackSeconds { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Settings/ConversationRelaySetting.cs b/src/Plugins/BotSharp.Plugin.Twilio/Settings/ConversationRelaySetting.cs index 5ce5fb7c5..9c3e533e6 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Settings/ConversationRelaySetting.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Settings/ConversationRelaySetting.cs @@ -144,6 +144,21 @@ public class ConversationRelaySetting /// public bool DropBackchannelTails { get; set; } + /// + /// Seconds past the moment everything said should have finished playing, with Twilio still saying the agent is speaking + /// and nothing cut into, before the session's playback is taken as stuck: the session drops its socket, Twilio connects + /// the call again, and the lines that never finished playing are said on the new session. Asks Twilio for its speaker + /// events (the events attribute with speaker-events); stops after . 0 turns it off. A negative value + /// takes a playback still going that long before its expected end as stuck: only to try the reconnect on a test call. + /// + /// + /// Why: on some calls Twilio said the agent started speaking and never that it stopped. Nothing said after that played, + /// and none of the caller's words came through, until they hung up; Twilio reported no error (dev calls 1c28cefa, + /// 46bc439c). How long a line takes is estimated from its length, on the slow side, so this is how much later than + /// even that the playback must still be going. + /// + public double StuckPlaybackSeconds { get; set; } + /// /// Settings for one agent's calls, by agent id, over the ones above /// @@ -172,6 +187,7 @@ public ConversationRelaySetting For(string? agentId) settings.MaxQuietWaitPrompts = agent.MaxQuietWaitPrompts ?? MaxQuietWaitPrompts; settings.ResumeAfterCutInSeconds = agent.ResumeAfterCutInSeconds ?? ResumeAfterCutInSeconds; settings.DropBackchannelTails = agent.DropBackchannelTails ?? DropBackchannelTails; + settings.StuckPlaybackSeconds = agent.StuckPlaybackSeconds ?? StuckPlaybackSeconds; return settings; } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/TwilioConversationRelayMiddleware.cs b/src/Plugins/BotSharp.Plugin.Twilio/TwilioConversationRelayMiddleware.cs index baf0442a0..bda7c2b98 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/TwilioConversationRelayMiddleware.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/TwilioConversationRelayMiddleware.cs @@ -47,10 +47,12 @@ public async Task Invoke(HttpContext httpContext) using var socket = await httpContext.WebSockets.AcceptWebSocketAsync(); var scopeFactory = services.GetRequiredService(); + var resumes = services.GetRequiredService(); var session = new ConversationRelaySession(socket, agentId, conversationId, settings, (message, callbacks) => SendCallerMessageAsync(scopeFactory, message, callbacks), _logger, - (line, label, saidUtc) => KeepSaidLineAsync(scopeFactory, agentId, conversationId, line, label, saidUtc)); + (line, label, saidUtc) => KeepSaidLineAsync(scopeFactory, agentId, conversationId, line, label, saidUtc), + resume => resumes.KeepAsync(conversationId, resume)); await services.GetRequiredService() .RunAsync(conversationId, () => session.RunAsync(httpContext.RequestAborted)); } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs b/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs index cd55abec9..3e96faa05 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs @@ -31,6 +31,7 @@ public void RegisterDI(IServiceCollection services, IConfiguration config) services.AddSingleton(sessionManager); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); services.AddHostedService(); services.AddTwilioRequestValidation(); services.AddScoped();