TV show sync

AGENTS.md

@@ -34,6 +34,7 @@ Anything else is replaced by an explicit command that downloads the official rel
 **IstarCI is recommended but optional, and runs from its package.**
 The CI is the GitHub Actions pipeline, which IstarCI only runs locally first.
 `task ci:setup`, `task ci` and `task ci:logs` use the installed `@devpro/istarci` package, and a clone of IstarCI is for developing IstarCI only (`task ci:from-clone`).
+A run starts by itself on every commit, so an agent never starts one: after committing it reads the result with `task ci` and, on a failure, `task ci:logs`.
 
 **The harness is Node.js and bash only.**
 Python scripts or code is not allowed.

Directory.Build.props

@@ -5,7 +5,7 @@
 
   <PropertyGroup>
     <!-- Tip: Edit this value to change the current MAJOR.MINOR.PATCH version -->
-    <VersionPrefix>2.4.3</VersionPrefix>
+    <VersionPrefix>2.4.4</VersionPrefix>
   </PropertyGroup>
 
   <Choose>

docs/reference-data.md

@@ -93,7 +93,7 @@ Each tick tries `ILeaseRepository.TryAcquireAsync("reference-sync", Environment.
 
 `ReferenceSyncService.SyncStaleReferencesAsync` is the single algorithm behind the loop and `POST /api/reference-data/sync-now`.
 The two differ only by the staleness windows, declared once in `ReferenceSyncWindows` (`Periodic`: 3 days for references, 7 for Explore; `Forced`: zero), guarded by `ReferenceSyncWindowsTest`.
-`sync-now?force=true` rechecks everything, and without it runs exactly the background tick.
+`sync-now?force=true` rechecks everything in full, skipping TMDB's change check (`ReferenceSyncWindows.FullFetch`), and without it runs exactly the background tick.
 One failing document never aborts the run, and one generic loop covers five one-line domain arms (`SyncDomainAsync`).
 
 **`I<X>ReferenceRepository.FindStaleAsync(cutoff, limit)` picks and orders the work** (`ReferenceStalenessQueries.cs`): never enriched first, then least recently enriched,
@@ -102,7 +102,9 @@ capped at `MaxDocumentsPerDomainPerPass` (500), so what a pass misses leads the
 **Gotcha: "never enriched" cannot come from the date comparison**, since `Lte(LastEnrichedAt, cutoff)` matches neither null nor missing.
 `Eq(field, null)` matches both and BSON sorts null first, proven only by the real-MongoDB `ReferenceStalenessRepositoryTest`, and `last_enriched_at` is indexed on all five collections.
 
-TV and movie refreshes pre-check TMDB's `/changes?start_date=...` and only bump `LastEnrichedAt` when nothing changed.
+TV and movie refreshes pre-check TMDB's `/changes?start_date=...&end_date=...` and only bump `LastEnrichedAt` when nothing changed.
+**Gotcha:** without `end_date` TMDB answers the changes of the `start_date` day only, and it rejects a range over 14 days,
+so a reference last enriched longer ago counts as changed and takes the full fetch (`TmdbClientTest`).
 IGDB, RAWG, Discogs and the book providers have no equivalent, so they always fetch in full past the cutoff.
 
 **Only the default video game provider is called on refresh**, unlike books, where a refresh checks every registered provider's key in `ExternalIds` (see [Providers](providers.md#multi-provider-domains)).

src/BlazorApp/Dockerfile

@@ -1,10 +1,10 @@
-FROM registry.suse.com/bci/dotnet-aspnet:10.0.13 AS base
+FROM registry.suse.com/bci/dotnet-aspnet:10.0.14 AS base
 USER $APP_UID
 WORKDIR /app
 EXPOSE 8080
 EXPOSE 8081
 
-FROM registry.suse.com/bci/dotnet-sdk:10.0.13 AS build
+FROM registry.suse.com/bci/dotnet-sdk:10.0.14 AS build
 ARG BUILD_CONFIGURATION=Release
 WORKDIR /src
 COPY ./Directory.*.props .

src/WebApi/Dockerfile

@@ -1,10 +1,10 @@
-FROM registry.suse.com/bci/dotnet-aspnet:10.0.13 AS base
+FROM registry.suse.com/bci/dotnet-aspnet:10.0.14 AS base
 USER $APP_UID
 WORKDIR /app
 EXPOSE 8080
 EXPOSE 8081
 
-FROM registry.suse.com/bci/dotnet-sdk:10.0.13 AS build
+FROM registry.suse.com/bci/dotnet-sdk:10.0.14 AS build
 ARG BUILD_CONFIGURATION=Release
 WORKDIR /src
 COPY ./Directory.*.props .

src/WebApi/ReferenceData/ReferenceDataAdminController.cs

@@ -18,7 +18,7 @@
 [ApiController]
 [Authorize(Policy = "AdminOnly")]
 [Route("api/reference-data")]
 public class ReferenceDataAdminController(
     ITvShowRepository tvShowRepository,
     IMovieRepository movieRepository,
     IBookRepository bookRepository,
@@ -265,7 +265,7 @@
     /// since a re-check across five domains easily exceeds a request's timeout.
     /// </summary>
     /// <param name="force">
-    /// <c>true</c> re-checks every reference document and rebuilds every Explore ranking, however recently enriched.
+    /// <c>true</c> re-checks every reference document in full and rebuilds every Explore ranking, however recently enriched.
     /// The default runs exactly what the background tick would (see <see cref="ReferenceSyncWindows.Periodic"/>), so a collection synced within the window reports zero checked.
     /// It is the cheap option, spending no provider calls on fresh documents.
     /// </param>
@@ -331,7 +331,7 @@
                 var scopedReconciliationService = scope.ServiceProvider.GetRequiredService<TvShowStatusReconciliationService>();
 
                 result = await scopedSyncService.SyncStaleReferencesAsync(
-                    windows.References, stage => scopedJobStore.UpdateStageAsync(jobId, stage), cancellationToken);
+                    windows.References, stage => scopedJobStore.UpdateStageAsync(jobId, stage), windows.FullFetch, cancellationToken);
                 // an on-demand "sync now" reconciles finished-show status too, so its result matches the periodic pass's.
                 result.FinishedShowsReopened = await scopedReconciliationService.ReconcileFinishedShowsAsync();
                 await scopedJobStore.UpdateStageAsync(jobId, ReferenceSyncStage.RefreshingExplore);

src/WebApi/ReferenceData/ReferenceEnrichmentService.TvShowsAndMovies.cs

@@ -531,13 +531,14 @@ private async Task WriteMovieLinkAsync(MovieReferenceModel reference, ReferenceL
     /// skipping the per-season fetches otherwise, and stamps <c>LastEnrichedAt</c> so the sync moves on.
     /// Unchanged for a reference without a TMDB id or one TMDB no longer holds.
     /// </summary>
-    public async Task<(TvShowReferenceModel Model, bool DataChanged)> RefreshTvShowReferenceAsync(TvShowReferenceModel reference, CancellationToken cancellationToken = default)
+    public async Task<(TvShowReferenceModel Model, bool DataChanged)> RefreshTvShowReferenceAsync(
+        TvShowReferenceModel reference, bool fullFetch = false, CancellationToken cancellationToken = default)
     {
         var tmdbId = reference.ExternalIds.GetValueOrDefault("tmdb");
         if (string.IsNullOrEmpty(tmdbId)) return (reference, false);
 
         // a full fetch while the reference has no ratings, so it backfills one (see RefreshMovieReferenceAsync)
-        if (reference.LastEnrichedAt is not null && reference.Ratings.Count > 0)
+        if (!fullFetch && reference.LastEnrichedAt is not null && reference.Ratings.Count > 0)
         {
             var changed = await tmdbClient.HasTvShowChangedSinceAsync(tmdbId, reference.LastEnrichedAt.Value, cancellationToken);
             if (!changed)
@@ -585,14 +586,15 @@ private async Task WriteMovieLinkAsync(MovieReferenceModel reference, ReferenceL
     /// <summary>
     /// Movie equivalent of <see cref="RefreshTvShowReferenceAsync"/>.
     /// </summary>
-    public async Task<(MovieReferenceModel Model, bool DataChanged)> RefreshMovieReferenceAsync(MovieReferenceModel reference, CancellationToken cancellationToken = default)
+    public async Task<(MovieReferenceModel Model, bool DataChanged)> RefreshMovieReferenceAsync(
+        MovieReferenceModel reference, bool fullFetch = false, CancellationToken cancellationToken = default)
     {
         var tmdbId = reference.ExternalIds.GetValueOrDefault("tmdb");
         if (string.IsNullOrEmpty(tmdbId)) return (reference, false);
 
         // A reference without ratings does the full fetch even when TMDB reports no change, or the change check would skip its backfill forever.
-        // A rated reference takes the cheap no-change shortcut.
-        if (reference.LastEnrichedAt is not null && reference.Ratings.Count > 0)
+        // A rated reference takes the cheap no-change shortcut, unless asked for a full fetch (see ReferenceSyncWindows.FullFetch).
+        if (!fullFetch && reference.LastEnrichedAt is not null && reference.Ratings.Count > 0)
         {
             var changed = await tmdbClient.HasMovieChangedSinceAsync(tmdbId, reference.LastEnrichedAt.Value, cancellationToken);
             if (!changed)

src/WebApi/ReferenceData/ReferenceSyncBackgroundService.cs

@@ -56,7 +56,7 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken)
                 jobId = await jobStore.CreateAsync(SystemOwnerId, ReferenceSyncStage.SyncingTvShows);
 
                 var syncService = scope.ServiceProvider.GetRequiredService<ReferenceSyncService>();
-                var result = await syncService.SyncStaleReferencesAsync(s_windows.References, cancellationToken: stoppingToken);
+                var result = await syncService.SyncStaleReferencesAsync(s_windows.References, fullFetch: s_windows.FullFetch, cancellationToken: stoppingToken);
 
                 // finished shows reconciled against the refreshed episode guides, in the same tick under the lease
                 var reconciliationService = scope.ServiceProvider.GetRequiredService<TvShowStatusReconciliationService>();

src/WebApi/ReferenceData/ReferenceSyncService.cs

@@ -27,20 +27,21 @@
     /// Refreshes the references whose <c>LastEnrichedAt</c> is unset or older than <paramref name="staleAfter"/>,
     /// oldest first, up to <see cref="MaxDocumentsPerDomainPerPass"/> per domain.
     /// A failing document is logged and skipped, never aborting the run.
+    /// <paramref name="fullFetch"/> skips TMDB's change check (see <see cref="ReferenceSyncWindows.FullFetch"/>), the other providers always fetching in full.
     /// </summary>
     public async Task<ReferenceSyncResultDto> SyncStaleReferencesAsync(TimeSpan staleAfter, Func<ReferenceSyncStage, Task>? onStageChanged = null,
-        CancellationToken cancellationToken = default)
+        bool fullFetch = false, CancellationToken cancellationToken = default)
     {
         var cutoff = DateTime.UtcNow - staleAfter;
         var result = new ReferenceSyncResultDto();
 
         // one loop for five domains, each arm supplying only which collection to read and which refresh to run
         await SyncDomainAsync(onStageChanged, ReferenceSyncStage.SyncingTvShows, cutoff,
-            tvShowReferenceRepository.FindStaleAsync, enrichmentService.RefreshTvShowReferenceAsync, r => r.Id,
+            tvShowReferenceRepository.FindStaleAsync, (r, ct) => enrichmentService.RefreshTvShowReferenceAsync(r, fullFetch, ct), r => r.Id,
             (checkedCount, updated) => (result.TvShowsChecked, result.TvShowsUpdated) = (checkedCount, updated), cancellationToken);
 
         await SyncDomainAsync(onStageChanged, ReferenceSyncStage.SyncingMovies, cutoff,
-            movieReferenceRepository.FindStaleAsync, enrichmentService.RefreshMovieReferenceAsync, r => r.Id,
+            movieReferenceRepository.FindStaleAsync, (r, ct) => enrichmentService.RefreshMovieReferenceAsync(r, fullFetch, ct), r => r.Id,
             (checkedCount, updated) => (result.MoviesChecked, result.MoviesUpdated) = (checkedCount, updated), cancellationToken);
 
         await SyncDomainAsync(onStageChanged, ReferenceSyncStage.SyncingBooks, cutoff,
@@ -61,7 +62,7 @@
     /// <summary>
     /// The sync of one domain: announce the stage, read the stalest page, refresh each document, and report the two counts.
     /// </summary>
     private async Task SyncDomainAsync<TReference>(
         Func<ReferenceSyncStage, Task>? onStageChanged,
         ReferenceSyncStage stage,
         DateTime cutoff,

src/WebApi/ReferenceData/ReferenceSyncWindows.cs

@@ -9,7 +9,10 @@ namespace Keeptrack.WebApi.ReferenceData;
 /// </summary>
 /// <param name="References">Applied to <see cref="ReferenceSyncService.SyncStaleReferencesAsync"/>.</param>
 /// <param name="Explore">Applied to <see cref="ExploreCatalogueRefreshService.RefreshAsync"/>.</param>
-public readonly record struct ReferenceSyncWindows(TimeSpan References, TimeSpan Explore)
+/// <param name="FullFetch">
+/// Skips TMDB's change check, which only sees changes after a reference's last enrichment, so forcing is the one way to recover a change it missed.
+/// </param>
+public readonly record struct ReferenceSyncWindows(TimeSpan References, TimeSpan Explore, bool FullFetch = false)
 {
     /// <summary>
     /// The windows of the 24h background tick.
@@ -21,9 +24,9 @@ public readonly record struct ReferenceSyncWindows(TimeSpan References, TimeSpan
     public static readonly ReferenceSyncWindows Periodic = new(TimeSpan.FromDays(3), TimeSpan.FromDays(7));
 
     /// <summary>
-    /// Checks everything again, however recently enriched, for the admin's "force" option.
+    /// Checks everything again in full, however recently enriched, for the admin's "force" option.
     /// </summary>
-    public static readonly ReferenceSyncWindows Forced = new(TimeSpan.Zero, TimeSpan.Zero);
+    public static readonly ReferenceSyncWindows Forced = new(TimeSpan.Zero, TimeSpan.Zero, FullFetch: true);
 
     /// <summary>
     /// <see cref="Forced"/> when <paramref name="force"/>, else the <see cref="Periodic"/> windows of the background pass.

src/WebApi/ReferenceData/TmdbClient.cs

@@ -90,7 +90,7 @@
         if (details is null) return null;
 
         var episodes = new List<TmdbEpisode>();
         foreach (var season in details.Seasons.Where(s => s.SeasonNumber > 0))
         {
             var seasonDetails = await http.GetFromJsonAsync<TmdbSeasonDetailsResponse>(
                 $"tv/{tmdbId}/season/{season.SeasonNumber}?api_key={ApiKey}", cancellationToken);
@@ -150,14 +150,25 @@
     }
 
     /// <summary>
-    /// TMDB's "changes" endpoint per id, unlike the bulk ones limited to the last days,
-    /// reports changes since any date, one cheap call rather than fetching the details and every season again.
+    /// TMDB rejects a <c>/changes</c> range longer than this, with status 20.
+    /// </summary>
+    private const int MaxChangesRangeDays = 14;
+
+    /// <summary>
+    /// TMDB's "changes" endpoint per id, one cheap call rather than fetching the details and every season again.
+    /// <para>
+    /// Without an <c>end_date</c> it answers the changes of the <c>start_date</c> day only, so the range is always closed at today.
+    /// A range TMDB would reject counts as changed, since a full fetch is the only safe answer.
+    /// </para>
     /// </summary>
     private async Task<bool> HasChangedSinceAsync(string resourceType, string tmdbId, DateTime since, CancellationToken cancellationToken)
     {
-        var startDate = since.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture);
+        var start = DateOnly.FromDateTime(since);
+        var end = DateOnly.FromDateTime(DateTime.UtcNow);
+        if (end.DayNumber - start.DayNumber > MaxChangesRangeDays) return true;
+
         var response = await http.GetFromJsonAsync<TmdbChangesResponse>(
-            $"{resourceType}/{tmdbId}/changes?api_key={ApiKey}&start_date={startDate}", cancellationToken);
+            $"{resourceType}/{tmdbId}/changes?api_key={ApiKey}&start_date={FormatDate(start)}&end_date={FormatDate(end)}", cancellationToken);
         return response?.Changes.Count > 0;
     }
 
@@ -168,6 +179,8 @@
 
     private static string Encode(string value) => HttpUtility.UrlEncode(value);
 
+    private static string FormatDate(DateOnly date) => date.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture);
+
     private static int? ParseYear(string? date) => ParseDate(date)?.Year;
 
     private static DateOnly? ParseDate(string? date) =>

test/WebApi.IntegrationTests/Resources/TvShowRepositoryTest.cs

@@ -0,0 +1,55 @@
+using System;
+using System.Linq;
+using System.Threading.Tasks;
+using AwesomeAssertions;
+using Keeptrack.Domain.Models;
+using Keeptrack.Domain.Repositories;
+using Keeptrack.WebApi.IntegrationTests.Hosting;
+using Microsoft.Extensions.DependencyInjection;
+using MongoDB.Bson;
+using Xunit;
+
+namespace Keeptrack.WebApi.IntegrationTests.Resources;
+
+/// <summary>
+/// Exercises <see cref="ITvShowRepository.FindFinishedLinkedShowsAsync"/> against real MongoDB, the query deciding which shows the status reconciliation may reopen.
+/// Null storage and the empty-string sentinel are MongoDB semantics a mocked repository cannot prove.
+/// </summary>
+public class TvShowRepositoryTest(KestrelWebAppFactory<Program> factory) : DatabaseTestBase(factory)
+{
+    [Fact]
+    public async Task FindFinishedLinkedShowsAsync_ReturnsOnlyFinishedShowsWithARealReference_AcrossOwners()
+    {
+        using var scope = Factory.Services.CreateScope();
+        var repository = scope.ServiceProvider.GetRequiredService<ITvShowRepository>();
+        var ownerId = $"owner-{Guid.NewGuid()}";
+        var otherOwnerId = $"owner-{Guid.NewGuid()}";
+        var referenceId = ObjectId.GenerateNewId().ToString();
+
+        var finished = await CreateShowAsync(repository, ownerId, TvShowStatus.Finished, referenceId);
+        var otherOwnersFinished = await CreateShowAsync(repository, otherOwnerId, TvShowStatus.Finished, referenceId);
+        var current = await CreateShowAsync(repository, ownerId, TvShowStatus.Current, referenceId);
+        var stopped = await CreateShowAsync(repository, ownerId, TvShowStatus.Stopped, referenceId);
+        var unset = await CreateShowAsync(repository, ownerId, null, referenceId);
+        var unlinked = await CreateShowAsync(repository, ownerId, TvShowStatus.Finished, null);
+        var emptySentinel = await CreateShowAsync(repository, ownerId, TvShowStatus.Finished, string.Empty);
+
+        var found = (await repository.FindFinishedLinkedShowsAsync()).Select(s => s.Id).ToList();
+
+        found.Should().Contain([finished.Id, otherOwnersFinished.Id]);
+        found.Should().NotContain([current.Id, stopped.Id, unset.Id, unlinked.Id, emptySentinel.Id]);
+    }
+
+    private async Task<TvShowModel> CreateShowAsync(ITvShowRepository repository, string ownerId, TvShowStatus? state, string? referenceId)
+    {
+        var created = await repository.CreateAsync(new TvShowModel
+        {
+            OwnerId = ownerId,
+            Title = $"The Diplomat {Guid.NewGuid()}",
+            State = state,
+            ReferenceId = referenceId
+        });
+        TrackCleanup(() => repository.DeleteAsync(created.Id!, ownerId));
+        return created;
+    }
+}