Reconciliate TV show status on new episodes

Directory.Build.props

@@ -5,7 +5,7 @@
 
   <PropertyGroup>
     <!-- Tip: Edit this value to change the current MAJOR.MINOR.PATCH version -->
-    <VersionPrefix>2.4.4</VersionPrefix>
+    <VersionPrefix>2.4.5</VersionPrefix>
   </PropertyGroup>
 
   <Choose>

src/WebApi/ReferenceData/ReferenceDataAdminController.cs

@@ -18,7 +18,7 @@
 [ApiController]
 [Authorize(Policy = "AdminOnly")]
 [Route("api/reference-data")]
 public class ReferenceDataAdminController(
     ITvShowRepository tvShowRepository,
     IMovieRepository movieRepository,
     IBookRepository bookRepository,
@@ -328,12 +328,9 @@
             if (!exploreOnly)
             {
                 var scopedSyncService = scope.ServiceProvider.GetRequiredService<ReferenceSyncService>();
-                var scopedReconciliationService = scope.ServiceProvider.GetRequiredService<TvShowStatusReconciliationService>();
 
                 result = await scopedSyncService.SyncStaleReferencesAsync(
                     windows.References, stage => scopedJobStore.UpdateStageAsync(jobId, stage), windows.FullFetch, cancellationToken);
-                // an on-demand "sync now" reconciles finished-show status too, so its result matches the periodic pass's.
-                result.FinishedShowsReopened = await scopedReconciliationService.ReconcileFinishedShowsAsync();
                 await scopedJobStore.UpdateStageAsync(jobId, ReferenceSyncStage.RefreshingExplore);
             }
 

src/WebApi/ReferenceData/ReferenceSyncBackgroundService.cs

@@ -58,10 +58,6 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken)
                 var syncService = scope.ServiceProvider.GetRequiredService<ReferenceSyncService>();
                 var result = await syncService.SyncStaleReferencesAsync(s_windows.References, fullFetch: s_windows.FullFetch, cancellationToken: stoppingToken);
 
-                // finished shows reconciled against the refreshed episode guides, in the same tick under the lease
-                var reconciliationService = scope.ServiceProvider.GetRequiredService<TvShowStatusReconciliationService>();
-                result.FinishedShowsReopened = await reconciliationService.ReconcileFinishedShowsAsync(stoppingToken);
-
                 // the Explore rankings ride the same tick on their own longer window (see ReferenceSyncWindows.Periodic)
                 var exploreRefreshService = scope.ServiceProvider.GetRequiredService<ExploreCatalogueRefreshService>();
                 result.ApplyExploreRefresh(await exploreRefreshService.RefreshAsync(s_windows.Explore, stoppingToken));

src/WebApi/ReferenceData/ReferenceSyncService.cs

@@ -5,15 +5,20 @@
 /// <summary>
 /// Keeps the shared reference collections up to date with their providers, since provider data (air dates, genres, covers, cast, ratings) changes after resolution.
 /// Shared by the periodic sync (<see cref="ReferenceSyncBackgroundService"/>) and the admin's "sync now", so both run the same logic.
+/// <para>
+/// Finished shows are reconciled right after the TV domain, since they depend on the TV guides alone,
+/// and a forced pass over the other four domains takes long enough that a page polling for the result is gone before it ends.
+/// </para>
 /// </summary>
 public class ReferenceSyncService(
     ITvShowReferenceRepository tvShowReferenceRepository,
     IMovieReferenceRepository movieReferenceRepository,
     IBookReferenceRepository bookReferenceRepository,
     IVideoGameReferenceRepository videoGameReferenceRepository,
     IAlbumReferenceRepository albumReferenceRepository,
     ReferenceEnrichmentService enrichmentService,
+    TvShowStatusReconciliationService tvShowStatusReconciliationService,
     ILogger<ReferenceSyncService> logger)
 {
     /// <summary>
     /// The most documents one pass refreshes per domain.
@@ -39,6 +44,15 @@
         await SyncDomainAsync(onStageChanged, ReferenceSyncStage.SyncingTvShows, cutoff,
             tvShowReferenceRepository.FindStaleAsync, (r, ct) => enrichmentService.RefreshTvShowReferenceAsync(r, fullFetch, ct), r => r.Id,
             (checkedCount, updated) => (result.TvShowsChecked, result.TvShowsUpdated) = (checkedCount, updated), cancellationToken);
+        try
+        {
+            result.FinishedShowsReopened = await tvShowStatusReconciliationService.ReconcileFinishedShowsAsync(cancellationToken);
+        }
+        // like a failing document, a failed reconciliation never costs the four domains still to sync
+        catch (Exception ex) when (ex is not OperationCanceledException)
+        {
+            logger.LogWarning(ex, "Failed to reconcile finished TV shows during the reference sync");
+        }
 
         await SyncDomainAsync(onStageChanged, ReferenceSyncStage.SyncingMovies, cutoff,
             movieReferenceRepository.FindStaleAsync, (r, ct) => enrichmentService.RefreshMovieReferenceAsync(r, fullFetch, ct), r => r.Id,
@@ -62,7 +76,7 @@
     /// <summary>
     /// The sync of one domain: announce the stage, read the stalest page, refresh each document, and report the two counts.
     /// </summary>
     private async Task SyncDomainAsync<TReference>(
         Func<ReferenceSyncStage, Task>? onStageChanged,
         ReferenceSyncStage stage,
         DateTime cutoff,

src/WebApi/ReferenceData/TvShowStatusReconciliationService.cs

@@ -13,7 +13,7 @@ namespace Keeptrack.WebApi.ReferenceData;
 /// so "finished" holds as of now and a regular pass checks it against the episode guide.
 /// When the linked reference lists an aired episode after the last one watched, the show goes back to <see cref="TvShowStatus.Current"/> and reappears in Watch Next.
 ///
-/// It runs right after <see cref="ReferenceSyncService"/>, in the same tick or "sync now", so it checks the freshly updated guides.
+/// <see cref="ReferenceSyncService"/> runs it right after the TV domain, in the same tick or "sync now", so it checks the freshly updated guides.
 /// It works on tenant data, kept apart from the reference refresh, whose only charter is the shared collections.
 ///
 /// Only finished shows are touched.

test/WebApi.IntegrationTests/Resources/TvShowStatusReconciliationTest.cs

@@ -0,0 +1,109 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using AwesomeAssertions;
+using Keeptrack.Domain.Models;
+using Keeptrack.Domain.Repositories;
+using Keeptrack.WebApi.IntegrationTests.Hosting;
+using Keeptrack.WebApi.ReferenceData;
+using Microsoft.Extensions.DependencyInjection;
+using Xunit;
+
+namespace Keeptrack.WebApi.IntegrationTests.Resources;
+
+/// <summary>
+/// <see cref="TvShowStatusReconciliationService"/> end to end against real MongoDB:
+/// the finished-show query, the batched reference and episode reads, and the status write each pass alone, so this pins them together.
+/// </summary>
+public class TvShowStatusReconciliationTest(KestrelWebAppFactory<Program> factory) : DatabaseTestBase(factory)
+{
+    private static readonly int[] s_episodesPerSeason = [8, 6, 8, 8];
+
+    [Fact]
+    public async Task ReconcileFinishedShowsAsync_ReopensAFinishedShowWatchedUpToTheLastSeasonFinale_OnceANewSeasonHasAired()
+    {
+        using var scope = Factory.Services.CreateScope();
+        var today = DateOnly.FromDateTime(DateTime.Today);
+        var ownerId = $"owner-{Guid.NewGuid()}";
+        var reference = await CreateReferenceAsync(scope, season => season == 4 ? today.AddDays(-13) : today.AddYears(season - 4));
+        var show = await CreateShowAsync(scope, ownerId, TvShowStatus.Finished, reference.Id!);
+        await RecordEpisodesUpToAsync(scope, ownerId, show.Id!, lastSeason: 3);
+
+        var reopened = await scope.ServiceProvider.GetRequiredService<TvShowStatusReconciliationService>().ReconcileFinishedShowsAsync(TestContext.Current.CancellationToken);
+
+        reopened.Should().BeGreaterThanOrEqualTo(1);
+        var stored = await scope.ServiceProvider.GetRequiredService<ITvShowRepository>().FindOneAsync(show.Id!, ownerId, TestContext.Current.CancellationToken);
+        stored!.State.Should().Be(TvShowStatus.Current);
+    }
+
+    [Fact]
+    public async Task ReconcileFinishedShowsAsync_LeavesAFinishedShowAlone_WhileTheNewSeasonHasNotAired()
+    {
+        using var scope = Factory.Services.CreateScope();
+        var today = DateOnly.FromDateTime(DateTime.Today);
+        var ownerId = $"owner-{Guid.NewGuid()}";
+        var reference = await CreateReferenceAsync(scope, season => season == 4 ? today.AddDays(30) : today.AddYears(season - 4));
+        var show = await CreateShowAsync(scope, ownerId, TvShowStatus.Finished, reference.Id!);
+        await RecordEpisodesUpToAsync(scope, ownerId, show.Id!, lastSeason: 3);
+
+        await scope.ServiceProvider.GetRequiredService<TvShowStatusReconciliationService>().ReconcileFinishedShowsAsync(TestContext.Current.CancellationToken);
+
+        var stored = await scope.ServiceProvider.GetRequiredService<ITvShowRepository>().FindOneAsync(show.Id!, ownerId, TestContext.Current.CancellationToken);
+        stored!.State.Should().Be(TvShowStatus.Finished);
+    }
+
+    private async Task<TvShowReferenceModel> CreateReferenceAsync(IServiceScope scope, Func<int, DateOnly> airDateOfSeason)
+    {
+        var episodes = new List<ReferenceEpisodeModel>();
+        for (var season = 1; season <= s_episodesPerSeason.Length; season++)
+        {
+            episodes.AddRange(Enumerable.Range(1, s_episodesPerSeason[season - 1]).Select(episode => new ReferenceEpisodeModel
+            {
+                SeasonNumber = season,
+                EpisodeNumber = episode,
+                Title = $"S{season}E{episode}",
+                AirDate = airDateOfSeason(season)
+            }));
+        }
+
+        var created = await scope.ServiceProvider.GetRequiredService<ITvShowReferenceRepository>().UpsertAsync(new TvShowReferenceModel
+        {
+            Title = "The Diplomat",
+            TitleNormalized = "the diplomat",
+            Year = 2023,
+            ExternalIds = new Dictionary<string, string> { ["tmdb"] = TestExternalId.New() },
+            Episodes = episodes
+        });
+        TrackDocument("tvshow_reference", created.Id);
+        return created;
+    }
+
+    private async Task<TvShowModel> CreateShowAsync(IServiceScope scope, string ownerId, TvShowStatus state, string referenceId)
+    {
+        var repository = scope.ServiceProvider.GetRequiredService<ITvShowRepository>();
+        var created = await repository.CreateAsync(new TvShowModel { OwnerId = ownerId, Title = "The Diplomat", Year = 2023, State = state, ReferenceId = referenceId });
+        TrackCleanup(() => repository.DeleteAsync(created.Id!, ownerId));
+        return created;
+    }
+
+    private async Task RecordEpisodesUpToAsync(IServiceScope scope, string ownerId, string showId, int lastSeason)
+    {
+        var repository = scope.ServiceProvider.GetRequiredService<IEpisodeRepository>();
+        for (var season = 1; season <= lastSeason; season++)
+        {
+            for (var episode = 1; episode <= s_episodesPerSeason[season - 1]; episode++)
+            {
+                var created = await repository.CreateAsync(new EpisodeModel
+                {
+                    OwnerId = ownerId,
+                    TvShowId = showId,
+                    SeasonNumber = season,
+                    EpisodeNumber = episode,
+                    WatchedAt = DateOnly.FromDateTime(DateTime.Today)
+                });
+                TrackCleanup(() => repository.DeleteAsync(created.Id!, ownerId));
+            }
+        }
+    }
+}

test/WebApi.UnitTests/ReferenceData/ReferenceSyncServiceTest.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
 using AwesomeAssertions;
@@ -12,6 +13,9 @@
 using Moq;
 using Xunit;
 
+// the Domain status enum, not the Contracts one the WebApi.Contracts.Dto using also imports
+using TvShowStatus = Keeptrack.Domain.Models.TvShowStatus;
+
 namespace Keeptrack.WebApi.UnitTests.ReferenceData;
 
 [Trait("Category", "UnitTests")]
@@ -28,6 +32,7 @@ public class ReferenceSyncServiceTest
     private readonly Mock<IBookRepository> _bookRepository = new();
     private readonly Mock<IVideoGameRepository> _videoGameRepository = new();
     private readonly Mock<IAlbumRepository> _albumRepository = new();
+    private readonly Mock<IEpisodeRepository> _episodeRepository = new();
 
     private ReferenceSyncService CreateService(FakeTmdbClient tmdbClient)
     {
@@ -46,10 +51,12 @@ private ReferenceSyncService CreateService(FakeTmdbClient tmdbClient)
             _bookReferenceRepository.Object, _videoGameReferenceRepository.Object, _albumReferenceRepository.Object,
             _tvShowRepository.Object, _movieRepository.Object, _bookRepository.Object, _videoGameRepository.Object, _albumRepository.Object,
             appSettingRepository.Object, new RatingSourceOptions(videoGameClients), NullLogger<ReferenceEnrichmentService>.Instance);
+        var reconciliationService = new TvShowStatusReconciliationService(
+            _tvShowRepository.Object, _episodeRepository.Object, _tvShowReferenceRepository.Object, NullLogger<TvShowStatusReconciliationService>.Instance);
         return new ReferenceSyncService(
             _tvShowReferenceRepository.Object, _movieReferenceRepository.Object,
             _bookReferenceRepository.Object, _videoGameReferenceRepository.Object, _albumReferenceRepository.Object,
-            enrichmentService, NullLogger<ReferenceSyncService>.Instance);
+            enrichmentService, reconciliationService, NullLogger<ReferenceSyncService>.Instance);
     }
 
     /// <summary>
@@ -161,6 +168,58 @@ public async Task SyncStaleReferencesAsync_TrustsTmdbsChangeCheck_OnThePeriodicP
         tmdbClient.ChangesRequested.Should().BeEquivalentTo(["42", "603"]);
     }
 
+    /// <summary>
+    /// A forced pass spends far longer on movies, books, video games and albums than on TV,
+    /// and finished shows only depend on the TV guides, so they are reopened before the next domain starts.
+    /// </summary>
+    [Fact]
+    public async Task SyncStaleReferencesAsync_ReopensFinishedShows_RightAfterTheTvShowsAndBeforeTheMovies()
+    {
+        _tvShowReferenceRepository.Setup(r => r.FindStaleAsync(It.IsAny<DateTime>(), It.IsAny<int>())).ReturnsAsync([]);
+        _movieReferenceRepository.Setup(r => r.FindStaleAsync(It.IsAny<DateTime>(), It.IsAny<int>())).ReturnsAsync([]);
+        _tvShowRepository.Setup(r => r.FindFinishedLinkedShowsAsync()).ReturnsAsync([
+            new TvShowModel { Id = "show-1", OwnerId = "owner", Title = "The Diplomat", State = TvShowStatus.Finished, ReferenceId = "reference-1" }
+        ]);
+        _episodeRepository.Setup(r => r.FindByShowIdsAsync("owner", It.IsAny<IReadOnlyCollection<string>>()))
+            .ReturnsAsync([new EpisodeModel { OwnerId = "owner", TvShowId = "show-1", SeasonNumber = 3, EpisodeNumber = 8 }]);
+        _tvShowReferenceRepository.Setup(r => r.FindByIdsAsync(It.IsAny<IReadOnlyCollection<string>>())).ReturnsAsync([
+            new TvShowReferenceModel
+            {
+                Id = "reference-1", Title = "The Diplomat", TitleNormalized = "the diplomat", ExternalIds = new Dictionary<string, string>(),
+                Episodes = [new ReferenceEpisodeModel { SeasonNumber = 4, EpisodeNumber = 1, Title = "S4E1", AirDate = new DateOnly(2026, 10, 15) }]
+            }
+        ]);
+        var reopenedBeforeMovies = false;
+        var service = CreateService(FakeTmdbClient.Empty());
+
+        var result = await service.SyncStaleReferencesAsync(TimeSpan.Zero, stage =>
+            {
+                if (stage == ReferenceSyncStage.SyncingMovies)
+                {
+                    reopenedBeforeMovies = _tvShowRepository.Invocations.Any(i => i.Method.Name == nameof(ITvShowRepository.UpdateAsync));
+                }
+                return Task.CompletedTask;
+            },
+            cancellationToken: TestContext.Current.CancellationToken);
+
+        reopenedBeforeMovies.Should().BeTrue();
+        result.FinishedShowsReopened.Should().Be(1);
+        _tvShowRepository.Verify(r => r.UpdateAsync("show-1", It.Is<TvShowModel>(s => s.State == TvShowStatus.Current), "owner", It.IsAny<CancellationToken>()), Times.Once);
+    }
+
+    [Fact]
+    public async Task SyncStaleReferencesAsync_StillSyncsTheOtherDomains_WhenTheReconciliationFails()
+    {
+        _tvShowReferenceRepository.Setup(r => r.FindStaleAsync(It.IsAny<DateTime>(), It.IsAny<int>())).ReturnsAsync([]);
+        _movieReferenceRepository.Setup(r => r.FindStaleAsync(It.IsAny<DateTime>(), It.IsAny<int>())).ReturnsAsync([]);
+        _tvShowRepository.Setup(r => r.FindFinishedLinkedShowsAsync()).ThrowsAsync(new InvalidOperationException("Simulated MongoDB failure."));
+        var service = CreateService(FakeTmdbClient.Empty());
+
+        await service.SyncStaleReferencesAsync(TimeSpan.Zero, cancellationToken: TestContext.Current.CancellationToken);
+
+        _albumReferenceRepository.Verify(r => r.FindStaleAsync(It.IsAny<DateTime>(), It.IsAny<int>()), Times.Once);
+    }
+
     [Fact]
     public async Task SyncStaleReferencesAsync_ContinuesPastAFailedReference_AndStillProcessesTheRest()
     {

test/WebApi.UnitTests/ReferenceData/TvShowStatusReconciliationServiceTest.cs

@@ -137,6 +137,58 @@ public async Task LeavesShowAlone_WhenTheOnlyUnwatchedEpisodeIsBeforeTheLastWatc
         reopened.Should().Be(0);
     }
 
+    [Fact]
+    public async Task LeavesShowAlone_WhenTheLatestWatchedIsTheFinale_EvenIfAnEarlierEpisodeWasRewatchedSince()
+    {
+        // the latest viewed episode is the furthest in (season, episode) order, never the most recent watch date
+        var rewatched = Episode("show-1", 1, 1);
+        rewatched.WatchedAt = new DateOnly(2026, 10, 1);
+        var finale = Episode("show-1", 3, 8);
+        finale.WatchedAt = new DateOnly(2025, 11, 1);
+        Setup(
+            [FinishedShow("show-1")],
+            [finale, rewatched],
+            Reference("ref-1", RefEpisode(1, 1, new DateOnly(2023, 4, 20)), RefEpisode(1, 2, new DateOnly(2023, 4, 20)), RefEpisode(3, 8, new DateOnly(2025, 10, 16))));
+        var service = CreateService();
+
+        var reopened = await service.ReconcileFinishedShowsAsync(TestContext.Current.CancellationToken);
+
+        reopened.Should().Be(0);
+    }
+
+    [Fact]
+    public async Task Reopens_FinishedShow_WhenANewSeasonAired_EvenIfAnEarlierEpisodeWasRewatchedSince()
+    {
+        var rewatched = Episode("show-1", 1, 1);
+        rewatched.WatchedAt = new DateOnly(2026, 10, 1);
+        var finale = Episode("show-1", 3, 8);
+        finale.WatchedAt = new DateOnly(2025, 11, 1);
+        Setup(
+            [FinishedShow("show-1")],
+            [finale, rewatched],
+            Reference("ref-1", RefEpisode(1, 1, new DateOnly(2023, 4, 20)), RefEpisode(3, 8, new DateOnly(2025, 10, 16)), RefEpisode(4, 1, new DateOnly(2026, 10, 15))));
+        var service = CreateService();
+
+        var reopened = await service.ReconcileFinishedShowsAsync(TestContext.Current.CancellationToken);
+
+        reopened.Should().Be(1);
+    }
+
+    [Fact]
+    public async Task Reopens_FinishedShow_WhenTheNewerEpisodeHasNoAirDate()
+    {
+        // TMDB leaves the date of an aired episode unset at times, and an unknown date is never read as "not aired yet"
+        Setup(
+            [FinishedShow("show-1")],
+            [Episode("show-1", 3, 8)],
+            Reference("ref-1", RefEpisode(3, 8, new DateOnly(2025, 10, 16)), RefEpisode(4, 1)));
+        var service = CreateService();
+
+        var reopened = await service.ReconcileFinishedShowsAsync(TestContext.Current.CancellationToken);
+
+        reopened.Should().Be(1);
+    }
+
     [Fact]
     public async Task LeavesShowAlone_WhenAlreadyCaughtUpWithTheReferenceGuide()
     {