diff --git a/src/Simplex/FileTransfer/Agent.hs b/src/Simplex/FileTransfer/Agent.hs index d83983363..c80a72fd1 100644 --- a/src/Simplex/FileTransfer/Agent.hs +++ b/src/Simplex/FileTransfer/Agent.hs @@ -207,7 +207,7 @@ runXFTPRcvWorker c srv Worker {doWork} = do rcvWorkerInternalError c rcvFileId rcvFileEntityId redirectEntityId_ (Just fileTmpPath) (INTERNAL "chunk has no replicas") (fc@RcvFileChunk {userId, rcvFileId, rcvFileEntityId, digest, fileTmpPath, replicas = replica@RcvFileChunkReplica {rcvChunkReplicaId, server, delay} : _}, approvedRelays, redirectEntityId_) -> do let ri' = maybe ri (\d -> ri {initialInterval = d, increaseAfter = 0}) delay - withRetryIntervalLimit xftpConsecutiveRetries ri' $ \delay' loop -> do + withRetryIntervalLimit xftpConsecutiveRetries ri' (readTVar $ networkEpoch c) $ \delay' loop -> do liftIO $ waitWhileSuspended c liftIO $ waitForUserNetwork c atomically $ incXFTPServerStat c userId srv downloadAttempts @@ -265,9 +265,9 @@ runXFTPRcvWorker c srv Worker {doWork} = do chunkReceived RcvFileChunk {replicas} = any received replicas -- The first call of action has n == 0, maxN is max number of retries -withRetryIntervalLimit :: forall m. MonadIO m => Int -> RetryInterval -> (Int64 -> m () -> m ()) -> m () -withRetryIntervalLimit maxN ri action = - withRetryIntervalCount ri $ \n delay loop -> +withRetryIntervalLimit :: forall m. MonadIO m => Int -> RetryInterval -> STM Int -> (Int64 -> m () -> m ()) -> m () +withRetryIntervalLimit maxN ri getEpoch action = + withRetryEpochCount ri getEpoch $ \n delay loop -> when (n < maxN) $ action delay loop retryOnError :: Text -> AM a -> AM a -> AgentErrorType -> AM a @@ -492,7 +492,7 @@ runXFTPSndPrepareWorker c Worker {doWork} = do triedHosts <- newTVarIO S.empty let AgentClient {xftpServers} = c userSrvCount <- liftIO $ length <$> TM.lookupIO userId xftpServers - withRetryIntervalCount (riFast ri) $ \n _ loop -> do + withRetryEpochCount (riFast ri) (readTVar $ networkEpoch c) $ \n _ loop -> do liftIO $ waitWhileSuspended c liftIO $ waitForUserNetwork c let triedAllSrvs = n > userSrvCount @@ -533,7 +533,7 @@ runXFTPSndWorker c srv Worker {doWork} = do SndFileChunk {sndFileId, sndFileEntityId, filePrefixPath, replicas = []} -> sndWorkerInternalError c sndFileId sndFileEntityId (Just filePrefixPath) (INTERNAL "chunk has no replicas") fc@SndFileChunk {userId, sndFileId, sndFileEntityId, filePrefixPath, digest, replicas = replica@SndFileChunkReplica {sndChunkReplicaId, server, delay} : _} -> do let ri' = maybe ri (\d -> ri {initialInterval = d, increaseAfter = 0}) delay - withRetryIntervalLimit xftpConsecutiveRetries ri' $ \delay' loop -> do + withRetryIntervalLimit xftpConsecutiveRetries ri' (readTVar $ networkEpoch c) $ \delay' loop -> do liftIO $ waitWhileSuspended c liftIO $ waitForUserNetwork c atomically $ incXFTPServerStat c userId srv uploadAttempts @@ -712,7 +712,7 @@ runXFTPDelWorker c srv Worker {doWork} = do where processDeletedReplica replica@DeletedSndChunkReplica {deletedSndChunkReplicaId, userId, server, chunkDigest, delay} = do let ri' = maybe ri (\d -> ri {initialInterval = d, increaseAfter = 0}) delay - withRetryIntervalLimit xftpConsecutiveRetries ri' $ \delay' loop -> do + withRetryIntervalLimit xftpConsecutiveRetries ri' (readTVar $ networkEpoch c) $ \delay' loop -> do liftIO $ waitWhileSuspended c liftIO $ waitForUserNetwork c atomically $ incXFTPServerStat c userId srv deleteAttempts diff --git a/src/Simplex/Messaging/Agent.hs b/src/Simplex/Messaging/Agent.hs index 5862f3b7c..5363d83e5 100644 --- a/src/Simplex/Messaging/Agent.hs +++ b/src/Simplex/Messaging/Agent.hs @@ -269,7 +269,7 @@ import Simplex.RemoteControl.Invitation import Simplex.RemoteControl.Types import System.Mem.Weak (deRefWeak) import UnliftIO.Async (mapConcurrently) -import UnliftIO.Concurrent (forkFinally, forkIO, killThread, mkWeakThreadId, threadDelay) +import UnliftIO.Concurrent (forkFinally, forkIO, forkIOWithUnmask, killThread, mkWeakThreadId, threadDelay) import qualified UnliftIO.Exception as E import UnliftIO.STM @@ -733,14 +733,39 @@ setNetworkConfig c@AgentClient {useNetworkConfig, proxySessTs} cfg' = do when changed $ liftIO $ reconnectAllServers c setUserNetworkInfo :: AgentClient -> UserNetworkInfo -> IO () -setUserNetworkInfo c@AgentClient {userNetworkInfo, userNetworkUpdated} ni = withAgentEnv' c $ do +setUserNetworkInfo c@AgentClient {userNetworkInfo, userNetworkUpdated, networkEventSeq} ni = withAgentEnv' c $ do ts' <- liftIO getCurrentTime i <- asks $ userOfflineDelay . config -- if network offline event happens in less than `userOfflineDelay` after the previous event, it is ignored - atomically . whenM ((isOnline ni ||) <$> notRecentlyChanged ts' i) $ do - writeTVar userNetworkInfo ni - writeTVar userNetworkUpdated $ Just ts' + -- mask, so that the thread restoring the deferred online network info is always forked + liftIO $ E.mask_ $ do + event <- atomically $ do + update <- (isOnline ni ||) <$> notRecentlyChanged ts' i + if update + then do + writeTVar userNetworkUpdated $ Just ts' + -- when the agent was offline the reported info is applied when the clients are closed, + -- so that the workers waiting for the network do not resume before then + wasOnline <- isOnline <$> readTVar userNetworkInfo + writeTVar userNetworkInfo $ if wasOnline || not (isOnline ni) then ni else ni {online = False} + modifyTVar' networkEventSeq (+ 1) + Just <$> readTVar networkEventSeq + else pure Nothing + -- The client reports online network when it changes. The connections to the servers made via + -- the previous network are likely to be dead, but the OS may only report it after several minutes. + forM_ event $ \e -> when (isOnline ni) $ void $ forkIOWithUnmask $ \unmask -> + unmask (closeClients e) `E.finally` networkOnline e where + -- skipped when a newer info superseded this one - it closes the clients itself, and the ones + -- that stay connected while the network is offline are closed by the next online info + closeClients e = do + whenCurrent $ closeNetworkClients c smpClients + whenCurrent $ closeNetworkClients c ntfClients + whenCurrent $ closeProtocolServerClients c xftpClients + where + whenCurrent = whenM (atomically $ isCurrent e) + networkOnline e = atomically $ whenM (isCurrent e) $ writeTVar userNetworkInfo ni >> modifyTVar' (networkEpoch c) (+ 1) + isCurrent e = (e ==) <$> readTVar networkEventSeq notRecentlyChanged ts' i = maybe True (\ts -> diffUTCTime ts' ts > i) <$> readTVar userNetworkUpdated @@ -749,6 +774,7 @@ reconnectAllServers c = do reconnectServerClients c smpClients reconnectServerClients c xftpClients reconnectServerClients c ntfClients + atomically $ modifyTVar' (networkEpoch c) (+ 1) -- | Register device notifications token registerNtfToken :: AgentClient -> NetworkRequestMode -> DeviceToken -> NotificationsMode -> AE NtfTknStatus @@ -2443,7 +2469,7 @@ runCommandProcessing c@AgentClient {subQ} connId server_ Worker {doWork} = do _ -> internalErr "command requires duplex connection" tryCommand action = tryMoveableCommand (action $> CCCompleted) tryAtomicCommand action = tryMoveableCommand (action $> CCDeleted) - tryMoveableCommand action = withRetryInterval2 ri $ \_ loop -> do + tryMoveableCommand action = withRetryEpoch2 ri (readTVar $ networkEpoch c) $ \_ loop -> do liftIO $ waitWhileSuspended c liftIO $ waitForUserNetwork c tryAllErrors action >>= \case @@ -2625,7 +2651,7 @@ runSmpQueueMsgDelivery c@AgentClient {subQ} sq@SndQueue {userId, connId, server, atomically $ endAgentOperation c AOMsgDelivery -- this operation begins in submitPendingMsg let mId = unId msgId ri' = maybe id updateRetryInterval2 msgRetryState ri - withRetryLock2 ri' qLock $ \riState loop -> do + withRetryLock2 ri' (readTVar $ networkEpoch c) qLock $ \riState loop -> do liftIO $ waitWhileSuspended c liftIO $ waitForUserNetwork c resp <- tryAllErrors $ case msgType of diff --git a/src/Simplex/Messaging/Agent/Client.hs b/src/Simplex/Messaging/Agent/Client.hs index d892fcad0..4e94580cb 100644 --- a/src/Simplex/Messaging/Agent/Client.hs +++ b/src/Simplex/Messaging/Agent/Client.hs @@ -21,6 +21,7 @@ {-# LANGUAGE TupleSections #-} {-# LANGUAGE TypeApplications #-} {-# LANGUAGE TypeFamilyDependencies #-} +{-# LANGUAGE TypeOperators #-} module Simplex.Messaging.Agent.Client ( AgentClient (..), @@ -147,6 +148,7 @@ module Simplex.Messaging.Agent.Client UserNetworkType (..), getFastNetworkConfig, waitForUserNetwork, + closeNetworkClients, isNetworkOnline, isOnline, throwWhenInactive, @@ -362,6 +364,10 @@ data AgentClient = AgentClient presetServers :: [SMPServer], userNetworkInfo :: TVar UserNetworkInfo, userNetworkUpdated :: TVar (Maybe UTCTime), + -- incremented when the clients are closed after the network changed, see withRetryEpoch + networkEpoch :: TVar Int, + -- incremented for each accepted network info, to identify the most recent one + networkEventSeq :: TVar Int, subscrConns :: TVar (Set ConnId), currentSubs :: TSessionSubs, removedSubs :: TMap (UserId, SMPServer) (TMap SMP.RecipientId SMPClientError), @@ -533,6 +539,8 @@ newAgentClient clientId InitialAgentServers {smp, ntf, xftp, entitlements, netCf useNetworkConfig <- newTVarIO (slowNetworkConfig netCfg, netCfg) userNetworkInfo <- newTVarIO $ UserNetworkInfo UNOther True userNetworkUpdated <- newTVarIO Nothing + networkEpoch <- newTVarIO 0 + networkEventSeq <- newTVarIO 0 subscrConns <- newTVarIO S.empty currentSubs <- SS.emptyIO removedSubs <- TM.emptyIO @@ -577,6 +585,8 @@ newAgentClient clientId InitialAgentServers {smp, ntf, xftp, entitlements, netCf presetServers, userNetworkInfo, userNetworkUpdated, + networkEpoch, + networkEventSeq, subscrConns, currentSubs, removedSubs, @@ -765,11 +775,17 @@ smpConnectClient c@AgentClient {smpClients, msgQ, proxySessTs, presetDomains} nm smp <- liftError (protocolClientError SMP $ B.unpack $ strEncode srv) $ do ts <- readTVarIO proxySessTs ExceptT $ getProtocolClient g nm tSess cfg' presetDomains (Just msgQ) ts $ smpClientDisconnected c tSess env v' prs - atomically $ SS.setSessionId tSess (sessionId $ thParams smp) $ currentSubs c - updateClientService service smp + -- the client removed from the map while it was connecting (see newProtocolClient) must not + -- replace the session or the service of the client that is current now + current <- atomically $ do + cur <- currentSessVar v' tSess smpClients + cur <$ when cur (SS.setSessionId tSess (sessionId $ thParams smp) $ currentSubs c) + when current $ updateClientService service smp pure SMPConnectedClient {connectedClient = smp, proxiedRelays = prs} updateClientService service smp = case (service, smpClientServiceId smp) of - (Just (_, serviceId_), Just serviceId) -> withStore' c $ \db -> do + (Just _, Just serviceId) -> withStore' c $ \db -> do + -- the id read before the handshake can be stale - this transaction can commit after another client's + serviceId_ <- (>>= snd) <$> getClientServiceCredentials db userId srv setClientServiceId db userId srv serviceId forM_ serviceId_ $ \sId -> when (sId /= serviceId) $ removeRcvServiceAssocs db userId srv (Just _, Nothing) -> withStore' c $ \db -> deleteClientService db userId srv -- e.g., server version downgrade @@ -795,9 +811,11 @@ smpClientDisconnected c@AgentClient {active, smpClients, smpProxiedRelays} tSess (subs, serviceSub_) <- SS.setSubsPending mode tSess sessId $ currentSubs c let qs = M.elems subs cs = nubOrd $ map qConnId qs - -- this removes proxied relays that this client created sessions to - destSrvs <- M.keys <$> readTVar prs - forM_ destSrvs $ \destSrv -> TM.delete (userId, destSrv, cId) smpProxiedRelays + -- this removes proxied relays that this client created sessions to, unless another client + -- replaced this one, e.g. when the network changed - those sessions are now its sessions + unlessM (TM.member tSess smpClients) $ do + destSrvs <- M.keys <$> readTVar prs + forM_ destSrvs $ \destSrv -> TM.delete (userId, destSrv, cId) smpProxiedRelays pure (qs, cs, serviceSub_) serverDown :: ([RcvQueueSub], [ConnId], Maybe ServiceSub) -> IO () @@ -830,7 +848,7 @@ resubscribeSMPSession c@AgentClient {smpSubWorkers, workerSeq} tSess = do atomically $ putTMVar (sessionVar v) a runSubWorker v = do ri <- asks $ reconnectInterval . config - withRetryForeground ri isForeground (isNetworkOnline c) $ \_ loop -> do + withRetryForeground ri isForeground (isNetworkOnline c) (readTVar $ networkEpoch c) $ \_ loop -> do pending_ <- atomically $ do pending@(pendingSubs, pendingSS) <- SS.getPendingSubs tSess $ currentSubs c if M.null pendingSubs && isNothing pendingSS @@ -952,8 +970,18 @@ newProtocolClient :: newProtocolClient c tSess@(userId, srv, entityId_) clients connectClient v = tryAllErrors (connectClient v) >>= \case Right client -> do + -- the client removed from the map while it was connecting, e.g. when the network changed, must + -- not be used - the requests made on it would wait for the responses that will not arrive; it is + -- published and rejected in one transaction, so that the caller always receives the one in the map + let e = BROKER (B.unpack $ strEncode srv) (NETWORK NEFailedError) + current <- atomically $ do + cur <- currentSessVar v tSess clients + cur <$ putTMVar (sessionVar v) (if cur then Right client else Left (e, Nothing)) + unless current $ do + -- closed in another thread, as closing an XFTP client waits for its thread to terminate + void $ liftIO $ forkIO $ closeProtocolServerClient (protocolClient client) `catchAll_` pure () + throwE e logInfo . decodeUtf8 $ "Agent connected to " <> showServer srv <> " (user " <> bshow userId <> maybe "" (" for entity " <>) entityId_ <> ")" - atomically $ putTMVar (sessionVar v) (Right client) liftIO $ nonBlockingWriteTBQueue (subQ c) ("", "", AEvt SAENone $ hostEvent CONNECT client) pure client Left e -> do @@ -1047,6 +1075,32 @@ reconnectServerClients :: ProtocolServerClient v err msg => AgentClient -> (Agen reconnectServerClients c clientsSel = readTVarIO (clientsSel c) >>= mapM_ (forkIO . closeClient_ c) +-- The pending requests of the clients that are closed fail before this function returns, so that +-- the workers that resume do not wait for the responses that will not arrive; the requests of +-- a client that did not terminate within the timeout are left to time out as before. +closeNetworkClients :: (ProtocolServerClient v err msg, ProtoClient msg ~ ProtocolClient v err msg) => AgentClient -> (AgentClient -> TMap (TransportSession msg) (ClientVar msg)) -> IO () +closeNetworkClients c clientsSel = do + closed <- readTVarIO (clientsSel c) >>= fmap catMaybes . mapM closeNetworkClient . M.toList + void $ 1000000 `timeout` atomically (mapM_ (\pc -> whenM (clientConnected pc) retry) closed) + -- the requests are failed after the client threads terminate, so that the responses that arrive + -- before that are processed as usual rather than as the responses to the failed requests + forM_ closed $ \pc -> unlessM (atomically $ clientConnected pc) $ failPendingRequests pc + where + closeNetworkClient (tSess, v) = + atomically (tryReadTMVar (sessionVar v) >>= \r -> r <$ removeUnconnected r) >>= \case + Just (Right client) -> do + let pc = protocolClient client + Just pc <$ forkIO (closeProtocolServerClient pc `catchAll_` pure ()) + _ -> pure Nothing + where + -- the client that failed to connect is removed so that the resumed workers do not get its cached + -- error, and the one that is connecting - so that they create a new client instead of waiting for + -- the connection made via the previous network; it is read and removed in one transaction, so that + -- it cannot publish itself in between - newProtocolClient then rejects and closes it + removeUnconnected = \case + Just (Right _) -> pure () + _ -> removeSessVar v tSess (clientsSel c) + reconnectSMPServer :: AgentClient -> UserId -> SMPServer -> IO () reconnectSMPServer c userId srv = do cs <- readTVarIO $ smpClients c diff --git a/src/Simplex/Messaging/Agent/NtfSubSupervisor.hs b/src/Simplex/Messaging/Agent/NtfSubSupervisor.hs index 7153abcbf..45f5617fd 100644 --- a/src/Simplex/Messaging/Agent/NtfSubSupervisor.hs +++ b/src/Simplex/Messaging/Agent/NtfSubSupervisor.hs @@ -456,7 +456,7 @@ retrySubActions _ [] _ = pure () retrySubActions c subs action = do v <- newTVarIO subs ri <- asks $ reconnectInterval . config - withRetryInterval ri $ \_ loop -> do + withRetryEpoch ri (readTVar $ networkEpoch c) $ \_ loop -> do liftIO $ waitWhileSuspended c liftIO $ waitForUserNetwork c subs' <- readTVarIO v @@ -563,7 +563,7 @@ runNtfTknDelWorker c srv Worker {doWork} = \nextTknToDelete -> do logInfo $ "runNtfTknDelWorker, nextTknToDelete " <> tshow nextTknToDelete ri <- asks $ reconnectInterval . config - withRetryInterval ri $ \_ loop -> do + withRetryEpoch ri (readTVar $ networkEpoch c) $ \_ loop -> do liftIO $ waitWhileSuspended c liftIO $ waitForUserNetwork c processTknToDelete nextTknToDelete `catchAllErrors` retryTmpError loop nextTknToDelete diff --git a/src/Simplex/Messaging/Agent/RetryInterval.hs b/src/Simplex/Messaging/Agent/RetryInterval.hs index 4ddc220da..f6447fd14 100644 --- a/src/Simplex/Messaging/Agent/RetryInterval.hs +++ b/src/Simplex/Messaging/Agent/RetryInterval.hs @@ -9,6 +9,9 @@ module Simplex.Messaging.Agent.RetryInterval RI2State (..), withRetryInterval, withRetryIntervalCount, + withRetryEpoch, + withRetryEpochCount, + withRetryEpoch2, withRetryForeground, withRetryInterval2, withRetryLock2, @@ -66,20 +69,50 @@ withRetryIntervalCount ri action = callAction 0 0 $ initialInterval ri let elapsed' = elapsed + delay callAction (n + 1) elapsed' $ nextRetryDelay elapsed' delay ri -withRetryForeground :: forall m a. MonadIO m => RetryInterval -> STM Bool -> STM Bool -> (Int64 -> m a -> m a) -> m a -withRetryForeground ri isForeground isOnline action = callAction 0 $ initialInterval ri +-- The delay restarts from the initial interval when the epoch changes (the agent increments it when the network changes) +-- while the loop waits; when it changed while the action was running the next attempt is made without any delay. +withRetryEpoch :: forall m a. MonadIO m => RetryInterval -> STM Int -> (Int64 -> m a -> m a) -> m a +withRetryEpoch ri getEpoch = withRetryEpochCount ri getEpoch . const + +withRetryEpochCount :: forall m a. MonadIO m => RetryInterval -> STM Int -> (Int -> Int64 -> m a -> m a) -> m a +withRetryEpochCount ri getEpoch action = callAction 0 0 $ initialInterval ri + where + callAction :: Int -> Int64 -> Int64 -> m a + callAction n elapsed delay = do + epoch <- atomically getEpoch + action n delay $ do + reset <- waitEpoch' epoch delay + let (elapsed', delay') + | reset = (0, initialInterval ri) + | otherwise = (elapsed + delay, nextRetryDelay elapsed' delay ri) + callAction (n + 1) elapsed' delay' + waitEpoch' = waitEpoch getEpoch + +-- returns True when the epoch changed before the delay expired +waitEpoch :: MonadIO m => STM Int -> Int -> Int64 -> m Bool +waitEpoch getEpoch epoch delay = do + -- limit delay to max Int value (~36 minutes on for 32 bit architectures) + d <- registerDelay $ fromIntegral $ min delay (fromIntegral (maxBound :: Int)) + atomically $ do + changed <- (epoch /=) <$> getEpoch + unlessM ((changed ||) <$> readTVar d) retry + pure changed + +withRetryForeground :: forall m a. MonadIO m => RetryInterval -> STM Bool -> STM Bool -> STM Int -> (Int64 -> m a -> m a) -> m a +withRetryForeground ri isForeground isOnline getEpoch action = callAction 0 $ initialInterval ri where callAction :: Int64 -> Int64 -> m a - callAction elapsed delay = action delay loop + callAction elapsed delay = action delay . loop =<< atomically getEpoch where - loop = do + loop epoch = do -- limit delay to max Int value (~36 minutes on for 32 bit architectures) d <- registerDelay $ fromIntegral $ min delay (fromIntegral (maxBound :: Int)) (wasForeground, wasOnline) <- atomically $ (,) <$> isForeground <*> isOnline reset <- atomically $ do foreground <- isForeground online <- isOnline - let reset = (not wasForeground && foreground) || (not wasOnline && online) + epochChanged <- (epoch /=) <$> getEpoch + let reset = (not wasForeground && foreground) || (not wasOnline && online) || epochChanged unlessM ((reset ||) <$> readTVar d) retry pure reset let (elapsed', delay') @@ -88,35 +121,55 @@ withRetryForeground ri isForeground isOnline action = callAction 0 $ initialInte callAction elapsed' delay' withRetryInterval2 :: forall m. MonadIO m => RetryInterval2 -> (RI2State -> (RetryIntervalMode -> m ()) -> m ()) -> m () -withRetryInterval2 = withRetryWait2 $ liftIO . threadDelay' +withRetryInterval2 = withRetryWait2 (pure ()) $ \_ _ delay -> False <$ liftIO (threadDelay' delay) -- This function allows action to toggle between slow and fast retry intervals. -withRetryLock2 :: forall m. MonadIO m => RetryInterval2 -> TMVar () -> (RI2State -> (RetryIntervalMode -> m ()) -> m ()) -> m () -withRetryLock2 ri lock = withRetryWait2 wait ri +-- The fast interval restarts on the epoch change, as in withRetryEpoch; once the action chose the slow +-- interval (recipient queue quota) the epoch change is not applied to the fast interval - neither the one +-- that happened during that action nor during the wait. +withRetryLock2 :: forall m. MonadIO m => RetryInterval2 -> STM Int -> TMVar () -> (RI2State -> (RetryIntervalMode -> m ()) -> m ()) -> m () +withRetryLock2 ri getEpoch lock = withRetryWait2 (atomically getEpoch) wait ri where - wait delay = do + wait mode epoch delay = do waiting <- newTVarIO True _ <- liftIO . forkIO $ do threadDelay' delay atomically $ whenM (readTVar waiting) $ void $ tryPutTMVar lock () atomically $ do - takeTMVar lock + reset <- (epochChanged <* tryTakeTMVar lock) `orElse` (False <$ takeTMVar lock) writeTVar waiting False + pure reset + where + epochChanged = case mode of + RIFast -> unlessM ((epoch /=) <$> getEpoch) retry >> pure True + RISlow -> retry -withRetryWait2 :: forall m. Monad m => (Int64 -> m ()) -> RetryInterval2 -> (RI2State -> (RetryIntervalMode -> m ()) -> m ()) -> m () -withRetryWait2 wait RetryInterval2 {riSlow, riFast} action = +-- As withRetryLock2, without the lock: the fast interval restarts on the epoch change, the slow one +-- (recipient queue quota) does not. +withRetryEpoch2 :: forall m. MonadIO m => RetryInterval2 -> STM Int -> (RI2State -> (RetryIntervalMode -> m ()) -> m ()) -> m () +withRetryEpoch2 ri getEpoch = withRetryWait2 (atomically getEpoch) wait ri + where + wait mode epoch delay = case mode of + RISlow -> False <$ liftIO (threadDelay' delay) + RIFast -> waitEpoch getEpoch epoch delay + +-- the value sampled before the action is passed to wait, so that a change that happened during the +-- action resets the interval as one during the wait does +withRetryWait2 :: forall m e. Monad m => m e -> (RetryIntervalMode -> e -> Int64 -> m Bool) -> RetryInterval2 -> (RI2State -> (RetryIntervalMode -> m ()) -> m ()) -> m () +withRetryWait2 sample wait RetryInterval2 {riSlow, riFast} action = callAction (0, initialInterval riSlow) (0, initialInterval riFast) where callAction :: (Int64, Int64) -> (Int64, Int64) -> m () - callAction slow fast = action (RI2State (snd slow) (snd fast)) loop + callAction slow fast = sample >>= \e -> action (RI2State (snd slow) (snd fast)) (loop e) where - loop = \case - RISlow -> run slow riSlow (`callAction` fast) - RIFast -> run fast riFast (callAction slow) - run (elapsed, delay) ri call = do - wait delay - let elapsed' = elapsed + delay - delay' = nextRetryDelay elapsed' delay ri + loop e = \case + RISlow -> run RISlow e slow riSlow (`callAction` fast) + RIFast -> run RIFast e fast riFast (callAction slow) + run mode e (elapsed, delay) ri call = do + reset <- wait mode e delay + let (elapsed', delay') + | reset = (0, initialInterval ri) + | otherwise = (elapsed + delay, nextRetryDelay elapsed' delay ri) call (elapsed', delay') nextRetryDelay :: Int64 -> Int64 -> RetryInterval -> Int64 diff --git a/src/Simplex/Messaging/Agent/TSessionSubs.hs b/src/Simplex/Messaging/Agent/TSessionSubs.hs index a1db48c9e..42a3dc01a 100644 --- a/src/Simplex/Messaging/Agent/TSessionSubs.hs +++ b/src/Simplex/Messaging/Agent/TSessionSubs.hs @@ -216,7 +216,9 @@ setSubsPending mode tSess@(uId, srv, connId_) sessId tss@(TSessionSubs ss) | entitySession == isJust connId_ = TM.lookup tSess ss >>= withSessSubs (`setSubsPending_` Nothing) | otherwise = - TM.lookupDelete tSess ss >>= withSessSubs setPendingChangeMode + -- the subscriptions are removed only when the session matches - a client that is no longer + -- the one in the map never set its session id, and its disconnection must not remove them + TM.lookup tSess ss >>= withSessSubs (\s -> TM.delete tSess ss >> setPendingChangeMode s) where entitySession = mode == TSMEntity sessEntId = if entitySession then Just else const Nothing diff --git a/src/Simplex/Messaging/Client.hs b/src/Simplex/Messaging/Client.hs index ba4bceec5..1fcc41377 100644 --- a/src/Simplex/Messaging/Client.hs +++ b/src/Simplex/Messaging/Client.hs @@ -35,6 +35,8 @@ module Simplex.Messaging.Client ProxiedRelay (..), getProtocolClient, closeProtocolClient, + failPendingRequests, + clientConnected, protocolClientServer, protocolClientServer', transportHost', @@ -151,7 +153,7 @@ import Data.Int (Int64) import Data.List (find, isSuffixOf) import Data.List.NonEmpty (NonEmpty (..)) import qualified Data.List.NonEmpty as L -import Data.Maybe (catMaybes, fromMaybe) +import Data.Maybe (catMaybes, fromMaybe, isJust) import Data.Text (Text) import qualified Data.Text as T import Data.Time.Clock (UTCTime (..), diffUTCTime, getCurrentTime) @@ -443,9 +445,12 @@ defaultNetworkConfig = transportClientConfig :: NetworkConfig -> NetworkRequestMode -> TransportHost -> Bool -> Maybe [ALPN] -> TransportClientConfig transportClientConfig NetworkConfig {socksProxy, socksMode, tcpConnectTimeout, tcpKeepAlive, logTLSErrors} nm host useSNI clientALPN = - TransportClientConfig {socksProxy = useSocksProxy socksMode, tcpConnectTimeout = tOut, tcpKeepAlive, logTLSErrors, clientCredentials = Nothing, clientALPN, useSNI} + TransportClientConfig {socksProxy = socks, tcpConnectTimeout = tOut, tcpKeepAlive = keepAlive, logTLSErrors, clientCredentials = Nothing, clientALPN, useSNI} where tOut = netTimeoutInt tcpConnectTimeout nm + socks = useSocksProxy socksMode + -- with SOCKS proxy the socket is connected to the proxy, and a slow circuit should not fail the connection + keepAlive = (\ka -> if isJust socks then ka {unackedDataTimeout = Nothing} else ka) <$> tcpKeepAlive socksProxy' = (\(SocksProxyWithAuth _ proxy) -> proxy) <$> socksProxy useSocksProxy SMAlways = socksProxy' useSocksProxy SMOnion = case host of @@ -652,10 +657,10 @@ getProtocolClient g nm transportSession@(_, srv, _) cfg@ProtocolClientConfig {qS writeTVar (connected c) True putTMVar cVar $ Right c' raceAny_ ([send c' th, process c', receive c' th] <> [monitor c' | smpPingInterval > 0]) - `E.finally` (failPendingRequests c >> disconnected c') + `E.finally` (atomically (writeTVar (connected c) False) >> failSentCommands c >> disconnected c') - failPendingRequests :: PClient v err msg -> IO () - failPendingRequests PClient {sentCommands} = atomically $ swapTVar sentCommands mempty >>= mapM_ failRequest + failSentCommands :: PClient v err msg -> IO () + failSentCommands PClient {sentCommands} = atomically $ swapTVar sentCommands mempty >>= mapM_ failRequest where failRequest Request {pending, responseVar} = whenM (swapTVar pending False) $ void $ tryPutTMVar responseVar $ Left $ PCENetworkError NEFailedError @@ -763,6 +768,13 @@ closeProtocolClient :: ProtocolClient v err msg -> IO () closeProtocolClient = mapM_ (deRefWeak >=> mapM_ killThread) . action {-# INLINE closeProtocolClient #-} +clientConnected :: ProtocolClient v err msg -> STM Bool +clientConnected ProtocolClient {client_ = PClient {connected}} = readTVar connected + +failPendingRequests :: ProtocolClient v err msg -> IO () +failPendingRequests ProtocolClient {client_ = PClient {sentCommands}} = + readTVarIO sentCommands >>= mapM_ (\Request {pending, responseVar} -> atomically $ whenM (swapTVar pending False) $ void $ tryPutTMVar responseVar $ Left $ PCENetworkError NEFailedError) + -- | SMP client error type. data ProtocolClientError err = -- | Correctly parsed SMP server ERR response. diff --git a/src/Simplex/Messaging/Session.hs b/src/Simplex/Messaging/Session.hs index 16ff3c934..67c9202f8 100644 --- a/src/Simplex/Messaging/Session.hs +++ b/src/Simplex/Messaging/Session.hs @@ -6,6 +6,7 @@ module Simplex.Messaging.Session ( SessionVar (..), getSessVar, removeSessVar, + currentSessVar, withGetSessVar, withGetSessVar', tryReadSessVar, @@ -44,6 +45,10 @@ removeSessVar v sessKey vs = Just v' | sessionVarId v == sessionVarId v' -> TM.delete sessKey vs _ -> pure () +-- the client is current while its session var is the one in the map +currentSessVar :: Ord k => SessionVar a -> k -> TMap k (SessionVar a) -> STM Bool +currentSessVar v sessKey vs = maybe False ((sessionVarId v ==) . sessionVarId) <$> TM.lookup sessKey vs + -- | Get or create a session var and route to onNew (newly created) or onExisting. The new-var -- branch is bracketed from the point of creation: if it is interrupted before filling the var -- (e.g. an async exception during connect), the still-empty var is dropped from the map so the diff --git a/src/Simplex/Messaging/Transport/KeepAlive.hs b/src/Simplex/Messaging/Transport/KeepAlive.hs index 1d34a27f0..42059f22d 100644 --- a/src/Simplex/Messaging/Transport/KeepAlive.hs +++ b/src/Simplex/Messaging/Transport/KeepAlive.hs @@ -11,6 +11,7 @@ module Simplex.Messaging.Transport.KeepAlive ) where import qualified Data.Aeson.TH as J +import Control.Monad (when) import Foreign.C (CInt (..)) import Network.Socket import Simplex.Messaging.Parsers (defaultJSON) @@ -18,16 +19,25 @@ import Simplex.Messaging.Parsers (defaultJSON) data KeepAliveOpts = KeepAliveOpts { keepIdle :: Int, keepIntvl :: Int, - keepCnt :: Int + keepCnt :: Int, + -- | A connection with unacknowledged data gets no keep-alive probes, so the OS fails it only when it + -- gives up retransmitting - ~15 minutes on Linux and Android. When this is set such a connection is + -- failed after the keep-alive detection time instead. Absent or False - the OS default. + unackedDataTimeout :: Maybe Bool } deriving (Eq, Show) +-- seconds after which a connection with no response to the probes is reported as failed +keepAliveDetectionTime :: KeepAliveOpts -> Int +keepAliveDetectionTime KeepAliveOpts {keepIdle, keepIntvl, keepCnt} = keepIdle + keepIntvl * keepCnt + defaultKeepAliveOpts :: KeepAliveOpts defaultKeepAliveOpts = KeepAliveOpts { keepIdle = 30, keepIntvl = 15, - keepCnt = 4 + keepCnt = 4, + unackedDataTimeout = Nothing } _SOL_TCP :: CInt @@ -53,8 +63,12 @@ _TCP_KEEPCNT = 16 #if defined(darwin_HOST_OS) foreign import capi "netinet/tcp.h value TCP_KEEPALIVE" _TCP_KEEPIDLE :: CInt + +foreign import capi "netinet/tcp.h value TCP_RXT_CONNDROPTIME" _TCP_RXT_CONNDROPTIME :: CInt #else foreign import capi "netinet/tcp.h value TCP_KEEPIDLE" _TCP_KEEPIDLE :: CInt + +foreign import capi "netinet/tcp.h value TCP_USER_TIMEOUT" _TCP_USER_TIMEOUT :: CInt #endif foreign import capi "netinet/tcp.h value TCP_KEEPINTVL" _TCP_KEEPINTVL :: CInt @@ -63,11 +77,22 @@ foreign import capi "netinet/tcp.h value TCP_KEEPCNT" _TCP_KEEPCNT :: CInt #endif +-- | seconds +setSocketUnackedDataTimeout :: Socket -> Int -> IO () +#if defined(mingw32_HOST_OS) +setSocketUnackedDataTimeout _ _ = pure () +#elif defined(darwin_HOST_OS) +setSocketUnackedDataTimeout sock = setSocketOption sock (SockOpt _SOL_TCP _TCP_RXT_CONNDROPTIME) +#else +setSocketUnackedDataTimeout sock t = setSocketOption sock (SockOpt _SOL_TCP _TCP_USER_TIMEOUT) (t * 1000) -- TCP_USER_TIMEOUT is in milliseconds +#endif + setSocketKeepAlive :: Socket -> KeepAliveOpts -> IO () -setSocketKeepAlive sock KeepAliveOpts {keepCnt, keepIdle, keepIntvl} = do +setSocketKeepAlive sock ka@KeepAliveOpts {keepCnt, keepIdle, keepIntvl, unackedDataTimeout} = do setSocketOption sock KeepAlive 1 setSocketOption sock (SockOpt _SOL_TCP _TCP_KEEPIDLE) keepIdle setSocketOption sock (SockOpt _SOL_TCP _TCP_KEEPINTVL) keepIntvl setSocketOption sock (SockOpt _SOL_TCP _TCP_KEEPCNT) keepCnt + when (unackedDataTimeout == Just True) $ setSocketUnackedDataTimeout sock $ keepAliveDetectionTime ka $(J.deriveJSON defaultJSON ''KeepAliveOpts) diff --git a/tests/AgentTests/FunctionalAPITests.hs b/tests/AgentTests/FunctionalAPITests.hs index 7ed0d17b8..efa67046e 100644 --- a/tests/AgentTests/FunctionalAPITests.hs +++ b/tests/AgentTests/FunctionalAPITests.hs @@ -94,6 +94,7 @@ import Simplex.Messaging.Agent.Client (ProtocolTestFailure (..), ProtocolTestSte import Simplex.Messaging.Agent.Env.SQLite (AgentConfig (..), Env (..), InitialAgentServers (..), createAgentStore) import Simplex.Messaging.Agent.Protocol hiding (CON, CONF, INFO, REQ, SENT) import qualified Simplex.Messaging.Agent.Protocol as A +import Simplex.Messaging.Agent.RetryInterval (RetryInterval (..), RetryInterval2 (..)) import Simplex.Messaging.Agent.Store (Connection' (..), SomeConn' (..), StoredRcvQueue (..)) import Simplex.Messaging.Agent.Store.AgentStore (deleteRatchetKeyHashesExpired, getConn, getConnIds, getRatchetX3dhKeys) import Simplex.Messaging.Agent.Store.Common (DBStore (..), withTransaction) @@ -628,6 +629,8 @@ functionalAPITimingTests ps = do describe "user network info" $ do it "should wait for user network" testWaitForUserNetwork it "should not reset online to offline if happens too quickly" testDoNotResetOnlineToOffline + it "should reconnect to servers when network changes" $ testNetworkChangeReconnect ps + it "should retry delivery and subscription immediately when network changes" $ testNetworkChangeRetry ps it "should resume multiple threads" testResumeMultipleThreads describe "SMP queue info" $ do it "server should respond with queue and subscription information" $ @@ -4992,9 +4995,52 @@ testWaitForUserNetwork = withAgent 1 aCfg initAgentServers testDB $ \a -> do (threadDelay 50000 >> setUserNetworkInfo a (UserNetworkInfo UNCellular True)) (networkDelay a 50000) noNetworkDelay a + -- the agent is disposed, so that the workers the network events resumed do not use the database + -- of the test that runs next - they share the file + disposeAgentClient a where aCfg = agentCfg {userNetworkInterval = 100000, userOfflineDelay = 0} +testNetworkChangeReconnect :: (HasCallStack, HasTestEnv) => (ASrvTransport, AStoreType) -> IO () +testNetworkChangeReconnect ps = + withAgentClientsCfg2 agentCfg {userOfflineDelay = 0} agentCfg $ \a b -> withSmpServer ps $ do + (aId, bId) <- runRight $ makeConnection a b + -- network type changed while online - connections to servers are re-established + setUserNetworkInfo a $ UserNetworkInfo UNCellular True + nGet a =##> \case ("", "", DOWN _ [c]) -> c == bId; _ -> False + nGet a =##> \case ("", "", UP _ [c]) -> c == bId; _ -> False + -- the same network type reported again (the client only reports it when the network changes) - reconnects + setUserNetworkInfo a $ UserNetworkInfo UNCellular True + nGet a =##> \case ("", "", DOWN _ [c]) -> c == bId; _ -> False + nGet a =##> \case ("", "", UP _ [c]) -> c == bId; _ -> False + setUserNetworkInfo a $ UserNetworkInfo UNNone False + noMessages a "no reconnection when offline" + setUserNetworkInfo a $ UserNetworkInfo UNWifi True + nGet a =##> \case ("", "", DOWN _ [c]) -> c == bId; _ -> False + nGet a =##> \case ("", "", UP _ [c]) -> c == bId; _ -> False + runRight_ $ exchangeGreetings a bId b aId + +testNetworkChangeRetry :: (HasCallStack, HasTestEnv) => (ASrvTransport, AStoreType) -> IO () +testNetworkChangeRetry ps = + withAgentClientsCfg2 aCfg agentCfg $ \a b -> do + (aId, bId) <- withSmpServerStoreLogOn ps testPort $ \_ -> runRight $ makeConnection a b + nGet a =##> \case ("", "", DOWN _ [c]) -> c == bId; _ -> False + nGet b =##> \case ("", "", DOWN _ [c]) -> c == aId; _ -> False + -- the first delivery attempt fails, and the delivery worker waits for the configured 60 seconds + 2 <- runRight $ sendMessage a bId SMP.noMsgFlags "hello" + threadDelay 500000 + withSmpServerStoreLogOn ps testPort $ \_ -> do + -- without network change event both the message delivery and the subscription + -- would be retried in 60 seconds, the test would fail with timeout + setUserNetworkInfo a $ UserNetworkInfo UNCellular True + runRight_ $ do + withUP a bId $ \case ("", c, SENT 2) -> c == bId; _ -> False + withUP b aId $ \case ("", c, Msg "hello") -> c == aId; _ -> False + ackMessage b aId 2 Nothing + where + slowRI = RetryInterval {initialInterval = 60_000000, increaseAfter = 0, maxInterval = 60_000000} + aCfg = agentCfg {reconnectInterval = slowRI, messageRetryInterval = RetryInterval2 {riSlow = slowRI, riFast = slowRI}} + testDoNotResetOnlineToOffline :: HasTestEnv => IO () testDoNotResetOnlineToOffline = withAgent 1 aCfg initAgentServers testDB $ \a -> do noNetworkDelay a @@ -5013,6 +5059,7 @@ testDoNotResetOnlineToOffline = withAgent 1 aCfg initAgentServers testDB $ \a -> setUserNetworkInfo a $ UserNetworkInfo UNWifi True setUserNetworkInfo a $ UserNetworkInfo UNNone False -- ingnored noNetworkDelay a + disposeAgentClient a where aCfg = agentCfg {userNetworkInterval = 100000, userOfflineDelay = 0.1} @@ -5034,6 +5081,7 @@ testResumeMultipleThreads = withAgent 1 aCfg initAgentServers testDB $ \a -> do let average = sum ts `div` fromIntegral (length ts) average < 3000000 `shouldBe` True maximum ts < 4000000 `shouldBe` True + disposeAgentClient a where aCfg = agentCfg {userOfflineDelay = 0} diff --git a/tests/CoreTests/RetryIntervalTests.hs b/tests/CoreTests/RetryIntervalTests.hs index f4b59ecc2..00f0d4a02 100644 --- a/tests/CoreTests/RetryIntervalTests.hs +++ b/tests/CoreTests/RetryIntervalTests.hs @@ -6,6 +6,7 @@ import Control.Concurrent (threadDelay) import Control.Concurrent.Async (concurrently_) import Control.Concurrent.STM import Control.Monad (when) +import Data.Int (Int64) import Data.Time.Clock (UTCTime, diffUTCTime, getCurrentTime, nominalDiffTimeToSeconds) import Simplex.Messaging.Agent.RetryInterval import Test.Hspec hiding (fit, it) @@ -16,13 +17,23 @@ retryIntervalTests = do describe "Retry interval with 2 modes and lock" $ do testRetryIntervalSameMode withRetryNewLock2 testRetryIntervalSwitchMode withRetryNewLock2 + testRetryIntervalEpochReset describe "Retry interval with 2 modes" $ do testRetryIntervalSameMode withRetryInterval2 testRetryIntervalSwitchMode withRetryInterval2 + describe "Retry interval with 2 modes and epoch" $ do + testRetryIntervalSameMode (`withRetryEpoch2` pure 0) + testRetryIntervalSwitchMode (`withRetryEpoch2` pure 0) describe "Foreground retry interval" $ do testRetryForeground testRetryToBackground testRetrySkipWhenForeground + testRetryForegroundEpoch + describe "Retry interval with epoch" $ do + testRetryEpochSameValue + testRetrySkipOnEpochChange + testRetryEpochChangedInAction + testRetryEpochKeepsCount testRI :: RetryInterval2 testRI = @@ -47,7 +58,7 @@ testFastRI = type WithRetry2 = RetryInterval2 -> (RI2State -> (RetryIntervalMode -> IO ()) -> IO ()) -> IO () withRetryNewLock2 :: WithRetry2 -withRetryNewLock2 ri action = newEmptyTMVarIO >>= \lock -> withRetryLock2 ri lock action +withRetryNewLock2 ri action = newEmptyTMVarIO >>= \lock -> withRetryLock2 ri (pure 0) lock action testRetryIntervalSameMode :: WithRetry2 -> Spec testRetryIntervalSameMode withRetry = @@ -97,6 +108,41 @@ testRetryIntervalSwitchMode withRetry = (200000, 200000) ] +testRetryIntervalEpochReset :: Spec +testRetryIntervalEpochReset = + it "should restart fast interval when epoch changes, but not slow interval" $ do + lock <- newEmptyTMVarIO + epoch <- newTVarIO (0 :: Int) + intervals <- newTVarIO [] + reportedIntervals <- newTVarIO [] + ts <- newTVarIO =<< getCurrentTime + concurrently_ + ( do + -- fast mode: waiting for 33750, restarts immediately at 65ms with the initial interval + threadDelay 65000 + atomically $ modifyTVar' epoch (+ 1) + -- slow mode: waiting for 20000 from 95ms, the change is ignored + threadDelay 40000 + atomically $ modifyTVar' epoch (+ 1) + ) + ( withRetryLock2 testRI (readTVar epoch) lock $ \(RI2State slow fast) loop -> do + ints <- addInterval intervals ts + atomically $ modifyTVar' reportedIntervals ((slow, fast) :) + when (length ints < 9) $ loop $ if length ints <= 6 then RIFast else RISlow + ) + (reverse <$> readTVarIO intervals) `shouldReturn` [0, 1, 1, 1, 2, 0, 1, 2, 2] + (reverse <$> readTVarIO reportedIntervals) + `shouldReturn` [ (20000, 10000), + (20000, 10000), + (20000, 15000), + (20000, 22500), + (20000, 33750), + (20000, 10000), + (20000, 10000), + (20000, 10000), + (30000, 10000) + ] + testRetryForeground :: Spec testRetryForeground = it "should increase elapased time and interval" $ do @@ -104,7 +150,7 @@ testRetryForeground = reportedIntervals <- newTVarIO [] ts <- newTVarIO =<< getCurrentTime let isForeground = pure True - withRetryForeground testFastRI isForeground (pure True) $ \delay loop -> do + withRetryForeground testFastRI isForeground (pure True) (pure 0) $ \delay loop -> do ints <- addInterval intervals ts atomically $ modifyTVar' reportedIntervals (delay :) when (length ints < 8) $ loop @@ -124,7 +170,7 @@ testRetryToBackground = threadDelay 250000 atomically $ writeTVar foreground False ) - ( withRetryForeground testFastRI (readTVar foreground) (pure True) $ \delay loop -> do + ( withRetryForeground testFastRI (readTVar foreground) (pure True) (pure 0) $ \delay loop -> do ints <- addInterval intervals ts atomically $ modifyTVar' reportedIntervals (delay :) when (length ints < 8) $ loop @@ -149,7 +195,89 @@ testRetrySkipWhenForeground = threadDelay 381250 atomically $ writeTVar foreground True ) - ( withRetryForeground testFastRI (readTVar foreground) (pure True) $ \delay loop -> do + ( withRetryForeground testFastRI (readTVar foreground) (pure True) (pure 0) $ \delay loop -> do + ints <- addInterval intervals ts + atomically $ modifyTVar' reportedIntervals (delay :) + when (length ints < 12) $ loop + ) + (reverse <$> readTVarIO intervals) `shouldReturn` [0, 1, 1, 1, 2, 0, 1, 1, 1, 2, 3, 1] + (reverse <$> readTVarIO reportedIntervals) + `shouldReturn` [10000, 10000, 15000, 22500, 33750, 10000, 10000, 15000, 22500, 33750, 40000, 10000] + +testRetryForegroundEpoch :: Spec +testRetryForegroundEpoch = + testEpochRestartsLoop "should repeat foreground loop as soon as epoch changes" $ \e -> + withRetryForeground testFastRI (pure True) (pure True) (readTVar e) + +testRetryEpochSameValue :: Spec +testRetryEpochSameValue = + it "should not change interval when epoch is written with the same value" $ do + intervals <- newTVarIO [] + reportedIntervals <- newTVarIO [] + ts <- newTVarIO =<< getCurrentTime + epoch <- newTVarIO (0 :: Int) + concurrently_ + ( do + threadDelay 50000 + atomically $ writeTVar epoch 0 + ) + ( withRetryEpoch testFastRI (readTVar epoch) $ \delay loop -> do + ints <- addInterval intervals ts + atomically $ modifyTVar' reportedIntervals (delay :) + when (length ints < 8) $ loop + ) + (reverse <$> readTVarIO intervals) `shouldReturn` [0, 1, 1, 1, 2, 3, 4, 4] + (reverse <$> readTVarIO reportedIntervals) + `shouldReturn` [10000, 10000, 15000, 22500, 33750, 40000, 40000, 40000] + +testRetryEpochChangedInAction :: Spec +testRetryEpochChangedInAction = + it "should restart interval when epoch changes while action runs" $ do + intervals <- newTVarIO [] + reportedIntervals <- newTVarIO [] + ts <- newTVarIO =<< getCurrentTime + epoch <- newTVarIO (0 :: Int) + withRetryEpoch testFastRI (readTVar epoch) $ \delay loop -> do + ints <- addInterval intervals ts + atomically $ modifyTVar' reportedIntervals (delay :) + when (length ints == 3) $ atomically $ modifyTVar' epoch (+ 1) + when (length ints < 6) $ loop + (reverse <$> readTVarIO intervals) `shouldReturn` [0, 1, 1, 0, 1, 1] + (reverse <$> readTVarIO reportedIntervals) + `shouldReturn` [10000, 10000, 15000, 10000, 10000, 15000] + +testRetryEpochKeepsCount :: Spec +testRetryEpochKeepsCount = + it "should not reset attempt count when epoch changes" $ do + counts <- newTVarIO ([] :: [Int]) + epoch <- newTVarIO (0 :: Int) + withRetryEpochCount testFastRI (readTVar epoch) $ \n _ loop -> do + ns <- atomically $ stateTVar counts $ \ns -> (n : ns, n : ns) + when (length ns == 2) $ atomically $ modifyTVar' epoch (+ 1) + when (length ns < 4) loop + (reverse <$> readTVarIO counts) `shouldReturn` [0, 1, 2, 3] + +testRetrySkipOnEpochChange :: Spec +testRetrySkipOnEpochChange = + testEpochRestartsLoop "should repeat loop as soon as epoch changes" $ \e -> + withRetryEpoch testFastRI (readTVar e) + +-- the epoch changes twice while the loop waits, so the interval restarts from the initial one twice +testEpochRestartsLoop :: String -> (TVar Int -> (Int64 -> IO () -> IO ()) -> IO ()) -> Spec +testEpochRestartsLoop name withRetry = + it name $ do + intervals <- newTVarIO [] + reportedIntervals <- newTVarIO [] + ts <- newTVarIO =<< getCurrentTime + epoch <- newTVarIO (0 :: Int) + concurrently_ + ( do + threadDelay 65000 + atomically $ modifyTVar' epoch (+ 1) + threadDelay 110000 + atomically $ modifyTVar' epoch (+ 1) + ) + ( withRetry epoch $ \delay loop -> do ints <- addInterval intervals ts atomically $ modifyTVar' reportedIntervals (delay :) when (length ints < 12) $ loop