From 446603b26c1509124471f0856080f560699ad37a Mon Sep 17 00:00:00 2001 From: Freesia Date: Tue, 3 Nov 2026 18:20:03 +0900 Subject: [PATCH 1/8] =?UTF-8?q?=E3=82=AA=E3=83=BC=E3=83=90=E3=83=BC?= =?UTF-8?q?=E3=83=AC=E3=82=A4=E9=9D=9E=E8=A1=A8=E7=A4=BA=E4=B8=AD=E3=81=AE?= =?UTF-8?q?=E8=BF=BD=E8=B7=A1=E3=82=92=E5=81=9C=E6=AD=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../Modules/Main/MainViewModelBase.cs | 192 ++++++++++++++---- 1 file changed, 158 insertions(+), 34 deletions(-) diff --git a/WindowTranslator/Modules/Main/MainViewModelBase.cs b/WindowTranslator/Modules/Main/MainViewModelBase.cs index 1b27e004..9198b439 100644 --- a/WindowTranslator/Modules/Main/MainViewModelBase.cs +++ b/WindowTranslator/Modules/Main/MainViewModelBase.cs @@ -22,10 +22,11 @@ namespace WindowTranslator.Modules.Main; [ObservableObject] public abstract partial class MainViewModelBase : IDisposable { - private readonly Subject captureRequests = new(); - private readonly Subject<(IEnumerable Texts, FilterContext Context)> trackingRequests = new(); - private readonly Subject<(IEnumerable Texts, FilterContext Context)> translationRequests = new(); - private readonly Subject> uiRequests = new(); + private readonly Subject captureRequests = new(); + private readonly Subject trackingRequests = new(); + private readonly Subject<(IEnumerable Texts, FilterContext Context, int VisibilityGeneration)> translationRequests = new(); + private readonly Subject<(IEnumerable Texts, int VisibilityGeneration)> uiRequests = new(); + private readonly object ocrTextTrackerLock = new(); private readonly IDisposable captureLoop; private readonly IDisposable trackingLoop; private readonly IDisposable translationLoop; @@ -66,11 +67,17 @@ public abstract partial class MainViewModelBase : IDisposable private bool isFirstCapture = true; private volatile bool disposedValue; + private int visibilityGeneration; public ObservableCollection OcrTexts { get; } = []; public string Font { get; } public double MousePointerHitTestPadding => this.mousePointerHitTestPadding; + private bool IsCurrentVisibleGeneration(int generation) + => !this.disposedValue + && this.OverlayVisible + && generation == Volatile.Read(ref this.visibilityGeneration); + public MainViewModelBase( IPresentationService presentationService, IOptionsSnapshot options, @@ -108,8 +115,8 @@ public MainViewModelBase( this.logger = logger; this.uiLoop = this.uiRequests .ObserveOn(new DispatcherSynchronizationContext(Application.Current.Dispatcher)) - .Where(_ => this.OverlayVisible) - .Subscribe(UpdateOcrTexts); + .Where(request => this.IsCurrentVisibleGeneration(request.VisibilityGeneration)) + .Subscribe(request => UpdateOcrTexts(request.Texts)); var captureInterval = TimeSpan.FromSeconds(options.Value.CaptureInterval); var captures = !this.isOneShotMode && captureInterval > TimeSpan.Zero ? this.captureRequests.ThrottleLast(captureInterval) @@ -119,40 +126,79 @@ public MainViewModelBase( .SubscribeAwait(RecognizeAsync, AwaitOperation.ThrottleFirstLast, configureAwait: false); this.trackingLoop = Disposable.Combine( Disposable.Create(() => this.trackingRequests.OnCompleted()), - this.ObservePropertyChanged(x => x.OverlayVisible).Where(v => v).ObserveOnThreadPool().Subscribe(_ => this.ocrTextTracker.Reset()), - Observable.Timer(TimeSpan.Zero, TimeSpan.FromMilliseconds(100), TimeProvider.System).Subscribe(_ => this.trackingRequests.OnNext(default)), + Observable.Timer(TimeSpan.Zero, TimeSpan.FromMilliseconds(100), TimeProvider.System) + .Where(_ => this.OverlayVisible) + .Subscribe(_ => this.trackingRequests.OnNext(new(TrackingSignalKind.Tick))), this.trackingRequests .Materialize() .Synchronize() - // タイマーの空通知では直前のOCR結果を再利用する。 - .Scan(default((IEnumerable Texts, FilterContext Context)), (previous, current) => + .Scan(new TrackingUpdate(null, null), (previous, current) => { - if (current.Value.Context is { } context) + if (current.Kind != NotificationKind.OnNext) { - if (previous.Context?.ImageSize != context.ImageSize) - { - this.ocrTextTracker.Reset(); - } - this.isFirstCapture |= this.isOneShotMode; + previous.Current?.Context.SoftwareBitmap.Dispose(); + return new(null, null); } - else if (current.Kind == NotificationKind.OnNext) + + switch (current.Value.Kind) { - return previous; + case TrackingSignalKind.Reset: + previous.Current?.Context.SoftwareBitmap.Dispose(); + this.ResetOcrTextTracker(); + return new(null, null); + case TrackingSignalKind.Observation: + var request = current.Value.Request!; + if (!this.IsCurrentVisibleGeneration(request.VisibilityGeneration)) + { + request.Context.SoftwareBitmap.Dispose(); + return new(previous.Current, null); + } + if (previous.Current?.Context.ImageSize != request.Context.ImageSize) + { + this.ResetOcrTextTracker(); + } + previous.Current?.Context.SoftwareBitmap.Dispose(); + this.isFirstCapture |= this.isOneShotMode; + return new(request, request); + case TrackingSignalKind.Tick: + if (previous.Current is not { } previousRequest) + { + return new(null, null); + } + if (this.IsCurrentVisibleGeneration(previousRequest.VisibilityGeneration)) + { + return new(previousRequest, previousRequest); + } + previousRequest.Context.SoftwareBitmap.Dispose(); + this.ResetOcrTextTracker(); + return new(null, null); + default: + throw new ArgumentOutOfRangeException(nameof(current), current.Value.Kind, null); } - previous.Context?.SoftwareBitmap.Dispose(); - return current.Value; }) - .Where(request => request.Context is not null) + .Where(update => update.Work is not null) + .Select(update => update.Work!) .SubscribeAwait(async (request, ct) => { + if (!this.IsCurrentVisibleGeneration(request.VisibilityGeneration)) + { + return; + } using var bitmap = request.Context.SoftwareBitmap.GetReadOnlyView(); var texts = await ProcessTrackingAsync(request, bitmap, ct); + if (!this.IsCurrentVisibleGeneration(request.VisibilityGeneration)) + { + return; + } if (this.isFirstCapture) { this.isFirstCapture = false; - this.uiRequests.OnNext(texts); + this.uiRequests.OnNext((texts, request.VisibilityGeneration)); + } + if (this.IsCurrentVisibleGeneration(request.VisibilityGeneration)) + { + this.translationRequests.OnNext((texts, request.Context with { SoftwareBitmap = bitmap.GetReadOnlyView(), }, request.VisibilityGeneration)); } - this.translationRequests.OnNext((texts, request.Context with { SoftwareBitmap = bitmap.GetReadOnlyView(), })); }, AwaitOperation.Drop, configureAwait: false, cancelOnCompleted: true)); this.translationLoop = this.translationRequests .SubscribeAwaitThrottleFirstLast(ProcessTranslationAsync, static r => r.Context.SoftwareBitmap.Dispose(), configureAwait: false); @@ -166,9 +212,11 @@ public MainViewModelBase( partial void OnOverlayVisibleChanged(bool value) { + var generation = Interlocked.Increment(ref this.visibilityGeneration); + this.trackingRequests.OnNext(new(TrackingSignalKind.Reset)); if (value) { - this.uiRequests.OnNext([]); + this.uiRequests.OnNext(([], generation)); this.isFirstCapture = !this.isOneShotMode; // Start capture when overlay becomes visible this.capture.StartCapture(this.processInfoStore.MainWindowHandle); @@ -183,6 +231,11 @@ partial void OnOverlayVisibleChanged(bool value) private async Task Capture_CapturedAsync(object? sender, CapturedEventArgs args) { + var generation = Volatile.Read(ref this.visibilityGeneration); + if (!this.IsCurrentVisibleGeneration(generation)) + { + return; + } if (this.isOneShotMode) { this.capture.StopCapture(); @@ -190,19 +243,24 @@ private async Task Capture_CapturedAsync(object? sender, CapturedEventArgs args) // フレームの寿命はイベント処理中だけなので、コピーの完了まで待つ。 var bitmap = await SoftwareBitmap.CreateCopyFromSurfaceAsync(args.Frame.Surface); - if (this.disposedValue) + if (this.disposedValue || !this.IsCurrentVisibleGeneration(generation)) { bitmap.Dispose(); return; } this.Width = bitmap.PixelWidth; this.Height = bitmap.PixelHeight; - this.captureRequests.OnNext(bitmap); + this.captureRequests.OnNext(new(bitmap, generation)); } - private async ValueTask RecognizeAsync(SoftwareBitmap capturedBitmap, CancellationToken cancellationToken) + private async ValueTask RecognizeAsync(CaptureRequest request, CancellationToken cancellationToken) { + var capturedBitmap = request.Bitmap; using var bitmap = new DisposeAction(capturedBitmap.Dispose); + if (!this.IsCurrentVisibleGeneration(request.VisibilityGeneration)) + { + return; + } IEnumerable texts; using (this.Recognizing.EnterBusy()) { @@ -267,7 +325,7 @@ private async ValueTask RecognizeAsync(SoftwareBitmap capturedBitmap, Cancellati } } cancellationToken.ThrowIfCancellationRequested(); - if (this.disposedValue) + if (this.disposedValue || !this.IsCurrentVisibleGeneration(request.VisibilityGeneration)) { return; } @@ -276,19 +334,48 @@ private async ValueTask RecognizeAsync(SoftwareBitmap capturedBitmap, Cancellati SoftwareBitmap = capturedBitmap.GetReadOnlyView(), ImageSize = new(capturedBitmap.PixelWidth, capturedBitmap.PixelHeight), }; - this.trackingRequests.OnNext((texts, context)); + this.trackingRequests.OnNext(new(TrackingSignalKind.Observation, new(texts, context, request.VisibilityGeneration))); } internal static IEnumerable SelectOcrTexts( IEnumerable observations, System.Drawing.Size imageSize, IOcrTextTracker tracker, bool isOneShotMode) => isOneShotMode ? observations : tracker.Update(observations, imageSize); - private async ValueTask> ProcessTrackingAsync((IEnumerable Texts, FilterContext Context) request, SoftwareBitmap bitmap, CancellationToken cancellationToken) + private void ResetOcrTextTracker() + { + lock (this.ocrTextTrackerLock) + { + this.ocrTextTracker.Reset(); + } + } + + private async ValueTask> ProcessTrackingAsync(TrackingRequest request, SoftwareBitmap bitmap, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); - var texts = SelectOcrTexts(request.Texts, request.Context.ImageSize, this.ocrTextTracker, this.isOneShotMode) + IEnumerable trackedTexts; + lock (this.ocrTextTrackerLock) + { + if (!this.IsCurrentVisibleGeneration(request.VisibilityGeneration)) + { + return []; + } + trackedTexts = SelectOcrTexts(request.Texts, request.Context.ImageSize, this.ocrTextTracker, this.isOneShotMode); + } + if (!this.IsCurrentVisibleGeneration(request.VisibilityGeneration)) + { + return []; + } + var texts = trackedTexts .Select(t => t with { FontSize = t.FontSize * this.fontScale }); + if (!this.IsCurrentVisibleGeneration(request.VisibilityGeneration)) + { + return []; + } texts = (await this.color.ConvertColorAsync(bitmap, texts)).ToArray(); + if (!this.IsCurrentVisibleGeneration(request.VisibilityGeneration)) + { + return []; + } texts = texts.Select(t => t switch { { TranslatedText: null } when this.cache.Contains(t.SourceText) => t with { TranslatedText = this.cache.Get(t.SourceText) }, @@ -298,11 +385,11 @@ private async ValueTask> ProcessTrackingAsync((IEnumerable return texts; } - private async ValueTask ProcessTranslationAsync((IEnumerable Texts, FilterContext Context) request, CancellationToken cancellationToken) + private async ValueTask ProcessTranslationAsync((IEnumerable Texts, FilterContext Context, int VisibilityGeneration) request, CancellationToken cancellationToken) { using var bitmap = request.Context.SoftwareBitmap; cancellationToken.ThrowIfCancellationRequested(); - if (this.disposedValue || !this.OverlayVisible) + if (!this.IsCurrentVisibleGeneration(request.VisibilityGeneration)) { return; } @@ -317,6 +404,10 @@ private async ValueTask ProcessTranslationAsync((IEnumerable Texts, Fi using var t = this.logger.LogDebugTime("翻訳前処理"); texts = await tmp.ToArrayAsync(cancellationToken); } + if (!this.IsCurrentVisibleGeneration(request.VisibilityGeneration)) + { + return; + } try { var requests = texts @@ -324,8 +415,16 @@ private async ValueTask ProcessTranslationAsync((IEnumerable Texts, Fi .ToArray(); if (requests.Length > 0) { + if (!this.IsCurrentVisibleGeneration(request.VisibilityGeneration)) + { + return; + } using var t = this.logger.LogDebugTime("翻訳"); var translated = await this.translator.TranslateAsync(requests).ConfigureAwait(false); + if (!this.IsCurrentVisibleGeneration(request.VisibilityGeneration)) + { + return; + } this.cache.AddRange(requests.Select(t => t.SourceText).Zip(translated)); var translatedIndex = 0; texts = texts.Select(t => t.TranslatedText is null @@ -351,6 +450,10 @@ private async ValueTask ProcessTranslationAsync((IEnumerable Texts, Fi await ShowErrorAsync(Resources.FaildOverlay, e, string.Empty); return; } + if (!this.IsCurrentVisibleGeneration(request.VisibilityGeneration)) + { + return; + } using (this.Filtering.EnterBusy()) { var tmp = texts.ToAsyncEnumerable(); @@ -362,7 +465,10 @@ private async ValueTask ProcessTranslationAsync((IEnumerable Texts, Fi texts = await tmp.ToArrayAsync(cancellationToken); } cancellationToken.ThrowIfCancellationRequested(); - this.uiRequests.OnNext(texts); + if (this.IsCurrentVisibleGeneration(request.VisibilityGeneration)) + { + this.uiRequests.OnNext((texts, request.VisibilityGeneration)); + } } private void UpdateOcrTexts(IEnumerable texts) @@ -380,6 +486,24 @@ private void UpdateOcrTexts(IEnumerable texts) } } + private sealed record CaptureRequest(SoftwareBitmap Bitmap, int VisibilityGeneration); + + private sealed record TrackingRequest( + IEnumerable Texts, + FilterContext Context, + int VisibilityGeneration); + + private sealed record TrackingSignal(TrackingSignalKind Kind, TrackingRequest? Request = null); + + private sealed record TrackingUpdate(TrackingRequest? Current, TrackingRequest? Work); + + private enum TrackingSignalKind + { + Tick, + Observation, + Reset, + } + private Task ShowErrorAsync(string message, Exception exception, string path) => Application.Current.Dispatcher.InvokeAsync(async () => { From 7bb6943a3609ab529f0974c869c6b514323fbb65 Mon Sep 17 00:00:00 2001 From: Freesia Date: Tue, 3 Nov 2026 18:38:55 +0900 Subject: [PATCH 2/8] =?UTF-8?q?=E8=BF=BD=E8=B7=A1=E5=87=A6=E7=90=86?= =?UTF-8?q?=E3=81=AE=E5=A4=89=E6=9B=B4=E3=82=92=E6=9C=80=E5=B0=8F=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../Modules/Main/MainViewModelBase.cs | 188 +++++------------- 1 file changed, 54 insertions(+), 134 deletions(-) diff --git a/WindowTranslator/Modules/Main/MainViewModelBase.cs b/WindowTranslator/Modules/Main/MainViewModelBase.cs index 9198b439..47bfd99f 100644 --- a/WindowTranslator/Modules/Main/MainViewModelBase.cs +++ b/WindowTranslator/Modules/Main/MainViewModelBase.cs @@ -22,11 +22,10 @@ namespace WindowTranslator.Modules.Main; [ObservableObject] public abstract partial class MainViewModelBase : IDisposable { - private readonly Subject captureRequests = new(); - private readonly Subject trackingRequests = new(); - private readonly Subject<(IEnumerable Texts, FilterContext Context, int VisibilityGeneration)> translationRequests = new(); - private readonly Subject<(IEnumerable Texts, int VisibilityGeneration)> uiRequests = new(); - private readonly object ocrTextTrackerLock = new(); + private readonly Subject captureRequests = new(); + private readonly Subject trackingRequests = new(); + private readonly Subject<(IEnumerable Texts, FilterContext Context)> translationRequests = new(); + private readonly Subject> uiRequests = new(); private readonly IDisposable captureLoop; private readonly IDisposable trackingLoop; private readonly IDisposable translationLoop; @@ -67,17 +66,11 @@ public abstract partial class MainViewModelBase : IDisposable private bool isFirstCapture = true; private volatile bool disposedValue; - private int visibilityGeneration; public ObservableCollection OcrTexts { get; } = []; public string Font { get; } public double MousePointerHitTestPadding => this.mousePointerHitTestPadding; - private bool IsCurrentVisibleGeneration(int generation) - => !this.disposedValue - && this.OverlayVisible - && generation == Volatile.Read(ref this.visibilityGeneration); - public MainViewModelBase( IPresentationService presentationService, IOptionsSnapshot options, @@ -115,8 +108,8 @@ public MainViewModelBase( this.logger = logger; this.uiLoop = this.uiRequests .ObserveOn(new DispatcherSynchronizationContext(Application.Current.Dispatcher)) - .Where(request => this.IsCurrentVisibleGeneration(request.VisibilityGeneration)) - .Subscribe(request => UpdateOcrTexts(request.Texts)); + .Where(_ => this.OverlayVisible) + .Subscribe(UpdateOcrTexts); var captureInterval = TimeSpan.FromSeconds(options.Value.CaptureInterval); var captures = !this.isOneShotMode && captureInterval > TimeSpan.Zero ? this.captureRequests.ThrottleLast(captureInterval) @@ -128,76 +121,59 @@ public MainViewModelBase( Disposable.Create(() => this.trackingRequests.OnCompleted()), Observable.Timer(TimeSpan.Zero, TimeSpan.FromMilliseconds(100), TimeProvider.System) .Where(_ => this.OverlayVisible) - .Subscribe(_ => this.trackingRequests.OnNext(new(TrackingSignalKind.Tick))), + .Subscribe(_ => this.trackingRequests.OnNext(new([], null))), this.trackingRequests .Materialize() .Synchronize() - .Scan(new TrackingUpdate(null, null), (previous, current) => + .Scan((TrackingRequest?)null, (previous, current) => { if (current.Kind != NotificationKind.OnNext) { - previous.Current?.Context.SoftwareBitmap.Dispose(); - return new(null, null); + previous?.Context?.SoftwareBitmap.Dispose(); + return null; } - switch (current.Value.Kind) + if (current.Value.Reset) + { + previous?.Context?.SoftwareBitmap.Dispose(); + this.ocrTextTracker.Reset(); + return null; + } + if (current.Value.Context is null) + { + if (!this.OverlayVisible) + { + previous?.Context?.SoftwareBitmap.Dispose(); + this.ocrTextTracker.Reset(); + return null; + } + return previous; + } + if (previous?.Context?.ImageSize != current.Value.Context.ImageSize) { - case TrackingSignalKind.Reset: - previous.Current?.Context.SoftwareBitmap.Dispose(); - this.ResetOcrTextTracker(); - return new(null, null); - case TrackingSignalKind.Observation: - var request = current.Value.Request!; - if (!this.IsCurrentVisibleGeneration(request.VisibilityGeneration)) - { - request.Context.SoftwareBitmap.Dispose(); - return new(previous.Current, null); - } - if (previous.Current?.Context.ImageSize != request.Context.ImageSize) - { - this.ResetOcrTextTracker(); - } - previous.Current?.Context.SoftwareBitmap.Dispose(); - this.isFirstCapture |= this.isOneShotMode; - return new(request, request); - case TrackingSignalKind.Tick: - if (previous.Current is not { } previousRequest) - { - return new(null, null); - } - if (this.IsCurrentVisibleGeneration(previousRequest.VisibilityGeneration)) - { - return new(previousRequest, previousRequest); - } - previousRequest.Context.SoftwareBitmap.Dispose(); - this.ResetOcrTextTracker(); - return new(null, null); - default: - throw new ArgumentOutOfRangeException(nameof(current), current.Value.Kind, null); + this.ocrTextTracker.Reset(); } + previous?.Context?.SoftwareBitmap.Dispose(); + this.isFirstCapture |= this.isOneShotMode; + return current.Value; }) - .Where(update => update.Work is not null) - .Select(update => update.Work!) + .Where(request => request is not null && request.Context is not null) .SubscribeAwait(async (request, ct) => { - if (!this.IsCurrentVisibleGeneration(request.VisibilityGeneration)) - { - return; - } - using var bitmap = request.Context.SoftwareBitmap.GetReadOnlyView(); + using var bitmap = request!.Context!.SoftwareBitmap.GetReadOnlyView(); var texts = await ProcessTrackingAsync(request, bitmap, ct); - if (!this.IsCurrentVisibleGeneration(request.VisibilityGeneration)) + if (!this.OverlayVisible) { return; } if (this.isFirstCapture) { this.isFirstCapture = false; - this.uiRequests.OnNext((texts, request.VisibilityGeneration)); + this.uiRequests.OnNext(texts); } - if (this.IsCurrentVisibleGeneration(request.VisibilityGeneration)) + if (this.OverlayVisible) { - this.translationRequests.OnNext((texts, request.Context with { SoftwareBitmap = bitmap.GetReadOnlyView(), }, request.VisibilityGeneration)); + this.translationRequests.OnNext((texts, request.Context! with { SoftwareBitmap = bitmap.GetReadOnlyView(), })); } }, AwaitOperation.Drop, configureAwait: false, cancelOnCompleted: true)); this.translationLoop = this.translationRequests @@ -212,11 +188,10 @@ public MainViewModelBase( partial void OnOverlayVisibleChanged(bool value) { - var generation = Interlocked.Increment(ref this.visibilityGeneration); - this.trackingRequests.OnNext(new(TrackingSignalKind.Reset)); + this.trackingRequests.OnNext(new([], null, Reset: true)); if (value) { - this.uiRequests.OnNext(([], generation)); + this.uiRequests.OnNext([]); this.isFirstCapture = !this.isOneShotMode; // Start capture when overlay becomes visible this.capture.StartCapture(this.processInfoStore.MainWindowHandle); @@ -231,8 +206,7 @@ partial void OnOverlayVisibleChanged(bool value) private async Task Capture_CapturedAsync(object? sender, CapturedEventArgs args) { - var generation = Volatile.Read(ref this.visibilityGeneration); - if (!this.IsCurrentVisibleGeneration(generation)) + if (!this.OverlayVisible) { return; } @@ -243,21 +217,20 @@ private async Task Capture_CapturedAsync(object? sender, CapturedEventArgs args) // フレームの寿命はイベント処理中だけなので、コピーの完了まで待つ。 var bitmap = await SoftwareBitmap.CreateCopyFromSurfaceAsync(args.Frame.Surface); - if (this.disposedValue || !this.IsCurrentVisibleGeneration(generation)) + if (this.disposedValue || !this.OverlayVisible) { bitmap.Dispose(); return; } this.Width = bitmap.PixelWidth; this.Height = bitmap.PixelHeight; - this.captureRequests.OnNext(new(bitmap, generation)); + this.captureRequests.OnNext(bitmap); } - private async ValueTask RecognizeAsync(CaptureRequest request, CancellationToken cancellationToken) + private async ValueTask RecognizeAsync(SoftwareBitmap capturedBitmap, CancellationToken cancellationToken) { - var capturedBitmap = request.Bitmap; using var bitmap = new DisposeAction(capturedBitmap.Dispose); - if (!this.IsCurrentVisibleGeneration(request.VisibilityGeneration)) + if (!this.OverlayVisible) { return; } @@ -325,7 +298,7 @@ private async ValueTask RecognizeAsync(CaptureRequest request, CancellationToken } } cancellationToken.ThrowIfCancellationRequested(); - if (this.disposedValue || !this.IsCurrentVisibleGeneration(request.VisibilityGeneration)) + if (this.disposedValue || !this.OverlayVisible) { return; } @@ -334,48 +307,19 @@ private async ValueTask RecognizeAsync(CaptureRequest request, CancellationToken SoftwareBitmap = capturedBitmap.GetReadOnlyView(), ImageSize = new(capturedBitmap.PixelWidth, capturedBitmap.PixelHeight), }; - this.trackingRequests.OnNext(new(TrackingSignalKind.Observation, new(texts, context, request.VisibilityGeneration))); + this.trackingRequests.OnNext(new(texts, context)); } internal static IEnumerable SelectOcrTexts( IEnumerable observations, System.Drawing.Size imageSize, IOcrTextTracker tracker, bool isOneShotMode) => isOneShotMode ? observations : tracker.Update(observations, imageSize); - private void ResetOcrTextTracker() - { - lock (this.ocrTextTrackerLock) - { - this.ocrTextTracker.Reset(); - } - } - private async ValueTask> ProcessTrackingAsync(TrackingRequest request, SoftwareBitmap bitmap, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); - IEnumerable trackedTexts; - lock (this.ocrTextTrackerLock) - { - if (!this.IsCurrentVisibleGeneration(request.VisibilityGeneration)) - { - return []; - } - trackedTexts = SelectOcrTexts(request.Texts, request.Context.ImageSize, this.ocrTextTracker, this.isOneShotMode); - } - if (!this.IsCurrentVisibleGeneration(request.VisibilityGeneration)) - { - return []; - } - var texts = trackedTexts + var texts = SelectOcrTexts(request.Texts, request.Context!.ImageSize, this.ocrTextTracker, this.isOneShotMode) .Select(t => t with { FontSize = t.FontSize * this.fontScale }); - if (!this.IsCurrentVisibleGeneration(request.VisibilityGeneration)) - { - return []; - } texts = (await this.color.ConvertColorAsync(bitmap, texts)).ToArray(); - if (!this.IsCurrentVisibleGeneration(request.VisibilityGeneration)) - { - return []; - } texts = texts.Select(t => t switch { { TranslatedText: null } when this.cache.Contains(t.SourceText) => t with { TranslatedText = this.cache.Get(t.SourceText) }, @@ -385,11 +329,11 @@ private async ValueTask> ProcessTrackingAsync(TrackingRequ return texts; } - private async ValueTask ProcessTranslationAsync((IEnumerable Texts, FilterContext Context, int VisibilityGeneration) request, CancellationToken cancellationToken) + private async ValueTask ProcessTranslationAsync((IEnumerable Texts, FilterContext Context) request, CancellationToken cancellationToken) { using var bitmap = request.Context.SoftwareBitmap; cancellationToken.ThrowIfCancellationRequested(); - if (!this.IsCurrentVisibleGeneration(request.VisibilityGeneration)) + if (this.disposedValue || !this.OverlayVisible) { return; } @@ -404,7 +348,7 @@ private async ValueTask ProcessTranslationAsync((IEnumerable Texts, Fi using var t = this.logger.LogDebugTime("翻訳前処理"); texts = await tmp.ToArrayAsync(cancellationToken); } - if (!this.IsCurrentVisibleGeneration(request.VisibilityGeneration)) + if (!this.OverlayVisible) { return; } @@ -415,13 +359,9 @@ private async ValueTask ProcessTranslationAsync((IEnumerable Texts, Fi .ToArray(); if (requests.Length > 0) { - if (!this.IsCurrentVisibleGeneration(request.VisibilityGeneration)) - { - return; - } using var t = this.logger.LogDebugTime("翻訳"); var translated = await this.translator.TranslateAsync(requests).ConfigureAwait(false); - if (!this.IsCurrentVisibleGeneration(request.VisibilityGeneration)) + if (!this.OverlayVisible) { return; } @@ -450,10 +390,6 @@ private async ValueTask ProcessTranslationAsync((IEnumerable Texts, Fi await ShowErrorAsync(Resources.FaildOverlay, e, string.Empty); return; } - if (!this.IsCurrentVisibleGeneration(request.VisibilityGeneration)) - { - return; - } using (this.Filtering.EnterBusy()) { var tmp = texts.ToAsyncEnumerable(); @@ -465,9 +401,9 @@ private async ValueTask ProcessTranslationAsync((IEnumerable Texts, Fi texts = await tmp.ToArrayAsync(cancellationToken); } cancellationToken.ThrowIfCancellationRequested(); - if (this.IsCurrentVisibleGeneration(request.VisibilityGeneration)) + if (this.OverlayVisible) { - this.uiRequests.OnNext((texts, request.VisibilityGeneration)); + this.uiRequests.OnNext(texts); } } @@ -486,23 +422,7 @@ private void UpdateOcrTexts(IEnumerable texts) } } - private sealed record CaptureRequest(SoftwareBitmap Bitmap, int VisibilityGeneration); - - private sealed record TrackingRequest( - IEnumerable Texts, - FilterContext Context, - int VisibilityGeneration); - - private sealed record TrackingSignal(TrackingSignalKind Kind, TrackingRequest? Request = null); - - private sealed record TrackingUpdate(TrackingRequest? Current, TrackingRequest? Work); - - private enum TrackingSignalKind - { - Tick, - Observation, - Reset, - } + private sealed record TrackingRequest(IEnumerable Texts, FilterContext? Context, bool Reset = false); private Task ShowErrorAsync(string message, Exception exception, string path) => Application.Current.Dispatcher.InvokeAsync(async () => From 890dd7eae5a22edd873efd348072f389dd80fe1f Mon Sep 17 00:00:00 2001 From: Freesia Date: Tue, 3 Nov 2026 18:45:37 +0900 Subject: [PATCH 3/8] =?UTF-8?q?=E8=BF=BD=E8=B7=A1=E5=81=9C=E6=AD=A2?= =?UTF-8?q?=E3=81=AE=E5=A4=89=E6=9B=B4=E3=82=92=E8=A6=81=E4=BB=B6=E7=AF=84?= =?UTF-8?q?=E5=9B=B2=E3=81=AB=E9=99=90=E5=AE=9A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../Modules/Main/MainViewModelBase.cs | 62 +++++++++---------- 1 file changed, 28 insertions(+), 34 deletions(-) diff --git a/WindowTranslator/Modules/Main/MainViewModelBase.cs b/WindowTranslator/Modules/Main/MainViewModelBase.cs index 47bfd99f..a2ecbe6d 100644 --- a/WindowTranslator/Modules/Main/MainViewModelBase.cs +++ b/WindowTranslator/Modules/Main/MainViewModelBase.cs @@ -121,47 +121,52 @@ public MainViewModelBase( Disposable.Create(() => this.trackingRequests.OnCompleted()), Observable.Timer(TimeSpan.Zero, TimeSpan.FromMilliseconds(100), TimeProvider.System) .Where(_ => this.OverlayVisible) - .Subscribe(_ => this.trackingRequests.OnNext(new([], null))), + .Subscribe(_ => this.trackingRequests.OnNext(default)), this.trackingRequests .Materialize() .Synchronize() - .Scan((TrackingRequest?)null, (previous, current) => + .Scan(default(TrackingRequest), (previous, current) => { if (current.Kind != NotificationKind.OnNext) { - previous?.Context?.SoftwareBitmap.Dispose(); - return null; + previous.Context?.SoftwareBitmap.Dispose(); + return default; } if (current.Value.Reset) { - previous?.Context?.SoftwareBitmap.Dispose(); + previous.Context?.SoftwareBitmap.Dispose(); this.ocrTextTracker.Reset(); - return null; + return default; + } + if (!this.OverlayVisible) + { + previous.Context?.SoftwareBitmap.Dispose(); + current.Value.Context?.SoftwareBitmap.Dispose(); + return default; } if (current.Value.Context is null) { - if (!this.OverlayVisible) - { - previous?.Context?.SoftwareBitmap.Dispose(); - this.ocrTextTracker.Reset(); - return null; - } return previous; } - if (previous?.Context?.ImageSize != current.Value.Context.ImageSize) + if (previous.Context?.ImageSize != current.Value.Context.ImageSize) { this.ocrTextTracker.Reset(); } - previous?.Context?.SoftwareBitmap.Dispose(); + previous.Context?.SoftwareBitmap.Dispose(); this.isFirstCapture |= this.isOneShotMode; return current.Value; }) - .Where(request => request is not null && request.Context is not null) + .Where(request => request.Context is not null) .SubscribeAwait(async (request, ct) => { - using var bitmap = request!.Context!.SoftwareBitmap.GetReadOnlyView(); - var texts = await ProcessTrackingAsync(request, bitmap, ct); + if (!this.OverlayVisible) + { + return; + } + var context = request.Context!; + using var bitmap = context.SoftwareBitmap.GetReadOnlyView(); + var texts = await ProcessTrackingAsync((request.Texts, context), bitmap, ct); if (!this.OverlayVisible) { return; @@ -171,10 +176,7 @@ public MainViewModelBase( this.isFirstCapture = false; this.uiRequests.OnNext(texts); } - if (this.OverlayVisible) - { - this.translationRequests.OnNext((texts, request.Context! with { SoftwareBitmap = bitmap.GetReadOnlyView(), })); - } + this.translationRequests.OnNext((texts, context with { SoftwareBitmap = bitmap.GetReadOnlyView(), })); }, AwaitOperation.Drop, configureAwait: false, cancelOnCompleted: true)); this.translationLoop = this.translationRequests .SubscribeAwaitThrottleFirstLast(ProcessTranslationAsync, static r => r.Context.SoftwareBitmap.Dispose(), configureAwait: false); @@ -206,10 +208,6 @@ partial void OnOverlayVisibleChanged(bool value) private async Task Capture_CapturedAsync(object? sender, CapturedEventArgs args) { - if (!this.OverlayVisible) - { - return; - } if (this.isOneShotMode) { this.capture.StopCapture(); @@ -217,7 +215,7 @@ private async Task Capture_CapturedAsync(object? sender, CapturedEventArgs args) // フレームの寿命はイベント処理中だけなので、コピーの完了まで待つ。 var bitmap = await SoftwareBitmap.CreateCopyFromSurfaceAsync(args.Frame.Surface); - if (this.disposedValue || !this.OverlayVisible) + if (this.disposedValue) { bitmap.Dispose(); return; @@ -230,10 +228,6 @@ private async Task Capture_CapturedAsync(object? sender, CapturedEventArgs args) private async ValueTask RecognizeAsync(SoftwareBitmap capturedBitmap, CancellationToken cancellationToken) { using var bitmap = new DisposeAction(capturedBitmap.Dispose); - if (!this.OverlayVisible) - { - return; - } IEnumerable texts; using (this.Recognizing.EnterBusy()) { @@ -298,7 +292,7 @@ private async ValueTask RecognizeAsync(SoftwareBitmap capturedBitmap, Cancellati } } cancellationToken.ThrowIfCancellationRequested(); - if (this.disposedValue || !this.OverlayVisible) + if (this.disposedValue) { return; } @@ -314,10 +308,10 @@ internal static IEnumerable SelectOcrTexts( IEnumerable observations, System.Drawing.Size imageSize, IOcrTextTracker tracker, bool isOneShotMode) => isOneShotMode ? observations : tracker.Update(observations, imageSize); - private async ValueTask> ProcessTrackingAsync(TrackingRequest request, SoftwareBitmap bitmap, CancellationToken cancellationToken) + private async ValueTask> ProcessTrackingAsync((IEnumerable Texts, FilterContext Context) request, SoftwareBitmap bitmap, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); - var texts = SelectOcrTexts(request.Texts, request.Context!.ImageSize, this.ocrTextTracker, this.isOneShotMode) + var texts = SelectOcrTexts(request.Texts, request.Context.ImageSize, this.ocrTextTracker, this.isOneShotMode) .Select(t => t with { FontSize = t.FontSize * this.fontScale }); texts = (await this.color.ConvertColorAsync(bitmap, texts)).ToArray(); texts = texts.Select(t => t switch @@ -422,7 +416,7 @@ private void UpdateOcrTexts(IEnumerable texts) } } - private sealed record TrackingRequest(IEnumerable Texts, FilterContext? Context, bool Reset = false); + private readonly record struct TrackingRequest(IEnumerable Texts, FilterContext? Context, bool Reset = false); private Task ShowErrorAsync(string message, Exception exception, string path) => Application.Current.Dispatcher.InvokeAsync(async () => From e16b2add7b07672246633690ef7a3f17a2af39d7 Mon Sep 17 00:00:00 2001 From: Freesia Date: Tue, 3 Nov 2026 22:14:23 +0900 Subject: [PATCH 4/8] =?UTF-8?q?=E9=9D=9E=E8=A1=A8=E7=A4=BA=E4=B8=AD?= =?UTF-8?q?=E3=81=AE=E8=BF=BD=E8=B7=A1=E3=81=A8=E5=BE=85=E6=A9=9F=E5=87=A6?= =?UTF-8?q?=E7=90=86=E3=82=92=E5=81=9C=E6=AD=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../OverlayVisibilityTests.cs | 625 ++++++++++++++++++ .../Modules/Main/MainViewModelBase.cs | 349 +++++++--- 2 files changed, 871 insertions(+), 103 deletions(-) create mode 100644 WindowTranslator.Tests/OverlayVisibilityTests.cs diff --git a/WindowTranslator.Tests/OverlayVisibilityTests.cs b/WindowTranslator.Tests/OverlayVisibilityTests.cs new file mode 100644 index 00000000..7a1e9f65 --- /dev/null +++ b/WindowTranslator.Tests/OverlayVisibilityTests.cs @@ -0,0 +1,625 @@ +using System.Diagnostics; +using System.Reflection; +using System.Runtime.ExceptionServices; +using System.Text; +using System.Windows; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.VisualStudio.Threading; +using R3; +using Windows.Graphics.Imaging; +using WindowTranslator.Modules; +using WindowTranslator.Modules.Capture; +using WindowTranslator.Modules.Main; +using WindowTranslator.Modules.Ocr; +using WindowTranslator.Stores; + +namespace WindowTranslator.Tests; + +public sealed class OverlayVisibilityTests +{ + private const string ChildProcessVariable = "WINDOW_TRANSLATOR_OVERLAY_TEST_CHILD"; + + [Fact] + public async Task 非表示中に追跡を止めて再表示後は新しい認識結果だけを表示する() + { + if (Environment.GetEnvironmentVariable(ChildProcessVariable) == "1") + { + await RunInStaAsync(); + return; + } + + // WPF の Application は同一プロセスで再作成できないため、既存テストと分離する。 + var start = new ProcessStartInfo(Environment.GetEnvironmentVariable("DOTNET_HOST_PATH") ?? "dotnet") + { + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true, + StandardOutputEncoding = Encoding.UTF8, + StandardErrorEncoding = Encoding.UTF8, + CreateNoWindow = true, + }; + start.ArgumentList.Add("vstest"); + start.ArgumentList.Add(typeof(OverlayVisibilityTests).Assembly.Location); + start.ArgumentList.Add("--TestCaseFilter:FullyQualifiedName~WindowTranslator.Tests.OverlayVisibilityTests"); + start.Environment[ChildProcessVariable] = "1"; + using var process = Process.Start(start) ?? throw new InvalidOperationException("テストプロセスを起動できません。"); + var output = process.StandardOutput.ReadToEndAsync(); + var error = process.StandardError.ReadToEndAsync(); + try + { + await process.WaitForExitAsync().WaitAsync(TimeSpan.FromSeconds(60)); + } + finally + { + if (!process.HasExited) + { + process.Kill(entireProcessTree: true); + } + } + Assert.True(process.ExitCode == 0, $"{await output}\n{await error}"); + } + + private static async Task RunInStaAsync() + { + var finished = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var thread = new Thread(() => + { + try + { + var application = new Application { ShutdownMode = ShutdownMode.OnExplicitShutdown }; + Exception? failure = null; + Fixture? afterShutdown = null; + SoftwareBitmap? pendingAfterShutdown = null; + _ = application.Dispatcher.InvokeAsync(async () => + { + try + { + await VerifyHiddenAsync(); + foreach (var stage in Enum.GetValues()) + { + await VerifyInFlightAsync(stage); + } + await VerifyInFlightAsync(Stage.Ocr, withoutContext: true); + await VerifyInFlightAsync(Stage.Ocr, failFirst: true); + await VerifyInFlightAsync(Stage.Translation, failFirst: true); + await VerifyHiddenDuringResetAsync(); + await VerifyHiddenDuringTrackingAsync(); + await VerifyHiddenBeforeTranslationAsync(); + await VerifyUiNotificationsAsync(); + await VerifyReentrantShowAsync(); + await VerifyReentrantHideAsync(); + await VerifyNestedVisibilityAsync(); + await VerifyPendingAsync(); + await VerifyDelayedCaptureAsync(); + await VerifyDisposeAsync(); + afterShutdown = new Fixture(captureInterval: 0.5); + pendingAfterShutdown = Bitmap(320); + afterShutdown.SubmitCapture(pendingAfterShutdown); + } + catch (Exception exception) + { + failure = exception; + } + finally + { + application.Shutdown(); + } + }); + application.Run(); + if (failure is not null) ExceptionDispatchInfo.Capture(failure).Throw(); + Task.Run(afterShutdown!.Model.Dispose).GetAwaiter().GetResult(); + Assert.True(IsDisposed(pendingAfterShutdown!)); + afterShutdown.DisposeAsync().AsTask().GetAwaiter().GetResult(); + finished.TrySetResult(); + } + catch (Exception exception) + { + finished.TrySetException(exception); + } + }) { IsBackground = true }; + thread.SetApartmentState(ApartmentState.STA); + thread.Start(); + await finished.Task.WaitAsync(TimeSpan.FromSeconds(45)); + } + + private static async Task VerifyHiddenAsync() + { + await using var fixture = new Fixture(); + SoftwareBitmap? tracked = null; + var hiddenTranslations = 0; + using var tracking = fixture.Tracking.Subscribe(request => + { + if (request.Context is { } context) tracked = context.SoftwareBitmap; + }); + using var translations = fixture.Translations.Subscribe(request => + { + if (request is not null && !fixture.Model.OverlayVisible) hiddenTranslations++; + }); + fixture.SubmitCapture(Bitmap(320)); + await WaitUntilAsync(() => fixture.Translator.Gate.Count > 0 && tracked is not null); + fixture.Model.OverlayVisible = false; + Assert.False(fixture.TrackingTimer.IsEnabled); + var trackerCount = fixture.Tracker.UpdateCount; + var colorCount = fixture.Color.Gate.Count; + await Task.Delay(400); + + Assert.Equal(trackerCount, fixture.Tracker.UpdateCount); + Assert.Equal(colorCount, fixture.Color.Gate.Count); + Assert.Equal(0, hiddenTranslations); + Assert.True(IsDisposed(tracked!)); + fixture.Model.OverlayVisible = true; + Assert.Empty(fixture.Model.OcrTexts); + fixture.SubmitCapture(Bitmap(321)); + await WaitUntilAsync(() => fixture.Model.OcrTexts.Any(text => text.SourceText == "new")); + Assert.DoesNotContain(fixture.Model.OcrTexts, text => text.SourceText == "old"); + Assert.True(fixture.Tracker.ResetCount >= 2); + } + + private static async Task VerifyInFlightAsync(Stage stage, bool withoutContext = false, bool failFirst = false) + { + await using var fixture = new Fixture(stage, withoutContext: withoutContext, failFirst: failFirst); + var gate = stage switch + { + Stage.Ocr => fixture.Ocr.Gate, + Stage.Color => fixture.Color.Gate, + _ => fixture.Translator.Gate, + }; + var shown = new List(); + fixture.Model.OcrTexts.CollectionChanged += (_, _) => + shown.AddRange(fixture.Model.OcrTexts.Select(text => text.SourceText)); + fixture.SubmitCapture(Bitmap(320)); + await gate.Started.Task.WaitAsync(TimeSpan.FromSeconds(5)); + fixture.Model.OverlayVisible = false; + fixture.Model.OverlayVisible = true; + shown.Clear(); + fixture.SubmitCapture(Bitmap(321)); + await Task.Delay(200); + Assert.False(gate.NextStarted.Task.IsCompleted); + gate.Release.TrySetResult(); + await gate.NextStarted.Task.WaitAsync(TimeSpan.FromSeconds(5)); + await WaitUntilAsync(() => fixture.Model.OcrTexts.Any(text => text.SourceText == "new")); + Assert.DoesNotContain("old", shown); + } + + private static async Task VerifyHiddenDuringResetAsync() + { + await using var fixture = new Fixture(delayReset: true); + fixture.SubmitCapture(Bitmap(320)); + await fixture.Tracker.ResetStarted.Task.WaitAsync(TimeSpan.FromSeconds(5)); + fixture.Model.OverlayVisible = false; + fixture.Tracker.ResetRelease.Set(); + await Task.Delay(250); + Assert.False(fixture.Tracker.Updated.Task.IsCompleted); + + fixture.Model.OverlayVisible = true; + fixture.SubmitCapture(Bitmap(321)); + await WaitUntilAsync(() => fixture.Model.OcrTexts.Any(text => text.SourceText == "new")); + } + + private static async Task VerifyHiddenDuringTrackingAsync() + { + await using var fixture = new Fixture(delayUpdate: true); + fixture.SubmitCapture(Bitmap(320)); + await fixture.Tracker.Updated.Task.WaitAsync(TimeSpan.FromSeconds(5)); + fixture.Model.OverlayVisible = false; + fixture.Tracker.UpdateRelease.Set(); + await Task.Delay(250); + Assert.Equal(0, fixture.Color.Gate.Count); + + fixture.Model.OverlayVisible = true; + fixture.SubmitCapture(Bitmap(321)); + await WaitUntilAsync(() => fixture.Model.OcrTexts.Any(text => text.SourceText == "new")); + } + + private static async Task VerifyHiddenBeforeTranslationAsync() + { + await using var fixture = new Fixture(delayFilter: true); + fixture.SubmitCapture(Bitmap(320)); + await fixture.Filter.Gate.Started.Task.WaitAsync(TimeSpan.FromSeconds(5)); + var pending = Bitmap(322); + fixture.Translations.OnNext(([new("pending", 0, 0, 100, 30, 20, false)], + new() { SoftwareBitmap = pending, ImageSize = new(322, 240) })); + fixture.Model.OverlayVisible = false; + Assert.True(IsDisposed(pending)); + fixture.Filter.Gate.Release.TrySetResult(); + await Task.Delay(250); + Assert.False(fixture.Translator.Gate.Started.Task.IsCompleted); + + fixture.Model.OverlayVisible = true; + fixture.SubmitCapture(Bitmap(321)); + await WaitUntilAsync(() => fixture.Translator.Gate.Started.Task.IsCompleted); + await WaitUntilAsync(() => fixture.Model.OcrTexts.Any(text => text.SourceText == "new")); + } + + private static async Task VerifyUiNotificationsAsync() + { + await using var fixture = new Fixture(multipleTexts: true); + var switched = false; + var addedAfterShow = new List(); + fixture.Model.OcrTexts.CollectionChanged += (_, change) => + { + if (change.NewItems is not { } items) return; + if (switched) + { + addedAfterShow.AddRange(items.Cast().Select(text => text.SourceText)); + } + else + { + switched = true; + fixture.Model.OverlayVisible = false; + fixture.Model.OverlayVisible = true; + } + }; + fixture.SubmitCapture(Bitmap(320)); + await WaitUntilAsync(() => switched); + await Task.Delay(100); + Assert.Empty(addedAfterShow); + Assert.Empty(fixture.Model.OcrTexts); + } + + private static async Task VerifyPendingAsync() + { + await using (var immediate = new Fixture()) + { + var image = Bitmap(322); + immediate.SubmitCapture(image); + immediate.Model.OverlayVisible = false; + await WaitUntilAsync(() => IsDisposed(image)); + } + + await using (var slow = new Fixture(captureInterval: (uint.MaxValue - 1) / 1000d)) + { + var pendingFrame = Bitmap(323); + slow.SubmitCapture(pendingFrame); + slow.Model.OverlayVisible = false; + Assert.True(IsDisposed(pendingFrame)); + } + + await using (var delayed = new Fixture(captureInterval: 0.1)) + { + delayed.SubmitCapture(Bitmap(320)); + Thread.Sleep(250); + delayed.Model.OverlayVisible = false; + delayed.Model.OverlayVisible = true; + delayed.SubmitCapture(Bitmap(321)); + await Task.Delay(30); + Assert.False(delayed.Ocr.Gate.Started.Task.IsCompleted); + await WaitUntilAsync(() => delayed.Model.OcrTexts.Any(text => text.SourceText == "new")); + } + + await using (var throttled = new Fixture(captureInterval: 0.5)) + { + var first = Bitmap(320); + var last = Bitmap(321); + throttled.SubmitCapture(first); + await Task.Delay(100); + throttled.SubmitCapture(last); + Assert.True(IsDisposed(first)); + await WaitUntilAsync(() => throttled.Model.OcrTexts.Any(text => text.SourceText == "new")); + Assert.DoesNotContain(throttled.Model.OcrTexts, text => text.SourceText == "old"); + var pendingFrame = Bitmap(323); + throttled.SubmitCapture(pendingFrame); + throttled.Model.OverlayVisible = false; + Assert.True(IsDisposed(pendingFrame)); + throttled.Model.OverlayVisible = true; + throttled.SubmitCapture(Bitmap(321)); + await WaitUntilAsync(() => throttled.Model.OcrTexts.Any(text => text.SourceText == "new")); + } + + await using var fixture = new Fixture(Stage.Ocr); + fixture.SubmitCapture(Bitmap(320)); + await fixture.Ocr.Gate.Started.Task.WaitAsync(TimeSpan.FromSeconds(5)); + var pending = Bitmap(322); + fixture.SubmitCapture(pending); + await Task.Delay(150); + fixture.Model.OverlayVisible = false; + Assert.True(IsDisposed(pending)); + fixture.Model.OverlayVisible = true; + fixture.Ocr.Gate.Release.TrySetResult(); + await Task.Delay(250); + Assert.False(fixture.Ocr.Gate.NextStarted.Task.IsCompleted); + } + + private static async Task VerifyDelayedCaptureAsync() + { + await using var fixture = new Fixture(); + var handler = fixture.Capture.CurrentHandler!; + fixture.Model.OverlayVisible = false; + fixture.Model.OverlayVisible = true; + await handler.InvokeAsync(fixture.Capture, null!); + await Task.Delay(150); + Assert.False(fixture.Ocr.Gate.Started.Task.IsCompleted); + } + + private static async Task VerifyReentrantShowAsync() + { + await using var fixture = new Fixture(Stage.Ocr); + var oldShown = false; + fixture.Model.OcrTexts.CollectionChanged += (_, _) => + oldShown |= fixture.Model.OcrTexts.Any(text => text.SourceText == "old"); + fixture.SubmitCapture(Bitmap(320)); + await fixture.Ocr.Gate.Started.Task.WaitAsync(TimeSpan.FromSeconds(5)); + + fixture.Capture.OnStopped = () => + { + fixture.Capture.OnStopped = null; + fixture.Model.OverlayVisible = true; + }; + fixture.Model.OverlayVisible = false; + fixture.SubmitCapture(Bitmap(321)); + fixture.Ocr.Gate.Release.TrySetResult(); + + await WaitUntilAsync(() => fixture.Model.OcrTexts.Any(text => text.SourceText == "new")); + Assert.False(oldShown); + } + + private static async Task VerifyDisposeAsync() + { + await using (var pending = new Fixture(captureInterval: 0.5)) + { + var image = Bitmap(320); + pending.SubmitCapture(image); + await Task.Run(pending.Model.Dispose).WaitAsync(TimeSpan.FromSeconds(5)); + Assert.True(IsDisposed(image)); + } + + await using (var running = new Fixture(Stage.Ocr)) + { + var image = Bitmap(320); + running.SubmitCapture(image); + await running.Ocr.Gate.Started.Task.WaitAsync(TimeSpan.FromSeconds(5)); + await Task.Run(running.Model.Dispose).WaitAsync(TimeSpan.FromSeconds(5)); + running.Ocr.Gate.Release.TrySetResult(); + await WaitUntilAsync(() => IsDisposed(image)); + } + } + + private static async Task VerifyReentrantHideAsync() + { + await using var fixture = new Fixture(); + fixture.Model.OcrTexts.Add(new TextRect("old", 0, 0, 100, 30, 20, false)); + fixture.Model.OverlayVisible = false; + var starts = fixture.Capture.StartCount; + fixture.Model.OcrTexts.CollectionChanged += (_, _) => fixture.Model.OverlayVisible = false; + + fixture.Model.OverlayVisible = true; + + Assert.False(fixture.Model.OverlayVisible); + Assert.Equal(starts, fixture.Capture.StartCount); + } + + private static async Task VerifyNestedVisibilityAsync() + { + await using var fixture = new Fixture(); + fixture.Model.OcrTexts.Add(new TextRect("old", 0, 0, 100, 30, 20, false)); + fixture.Model.OverlayVisible = false; + var starts = fixture.Capture.StartCount; + var toggled = false; + fixture.Model.OcrTexts.CollectionChanged += (_, _) => + { + if (toggled) return; + toggled = true; + fixture.Model.OverlayVisible = false; + fixture.Model.OverlayVisible = true; + }; + fixture.Model.OcrTexts.CollectionChanged += (_, _) => { }; + + fixture.Model.OverlayVisible = true; + await Task.Delay(50); + + Assert.True(fixture.Model.OverlayVisible); + Assert.Equal(starts + 1, fixture.Capture.StartCount); + } + + private static SoftwareBitmap Bitmap(int width) => new(BitmapPixelFormat.Bgra8, width, 240); + + private static bool IsDisposed(SoftwareBitmap bitmap) + { + try + { + using var view = bitmap.GetReadOnlyView(); + return false; + } + catch (Exception exception) when (exception is ObjectDisposedException || exception.HResult == unchecked((int)0x80000013)) + { + return true; + } + } + + private static async Task WaitUntilAsync(Func condition) + { + var timeout = Stopwatch.StartNew(); + while (!condition() && timeout.Elapsed < TimeSpan.FromSeconds(5)) + { + await Task.Delay(20); + } + Assert.True(condition(), "期待した処理が時間内に完了しませんでした。"); + } + + private enum Stage { Ocr, Color, Translation } + + private sealed class Fixture : System.IAsyncDisposable + { + private readonly OcrTraceRecorder recorder = new(NullLogger.Instance); + private readonly Action submitCapture; + + public Fixture(Stage? delayed = null, double captureInterval = 0, bool delayReset = false, bool delayFilter = false, bool multipleTexts = false, bool withoutContext = false, bool failFirst = false, bool delayUpdate = false) + { + this.Ocr = new(delayed == Stage.Ocr, multipleTexts, failFirst && delayed == Stage.Ocr); + this.Color = new(delayed == Stage.Color); + this.Translator = new(delayed == Stage.Translation, failFirst && delayed == Stage.Translation); + this.Tracker = new(delayReset, delayUpdate); + this.Filter = new(delayFilter); + var context = SynchronizationContext.Current; + try + { + if (withoutContext) SynchronizationContext.SetSynchronizationContext(null); + this.Model = new OverlayMainViewModel( + null!, new Snapshot(new() { CaptureInterval = captureInterval }), new ProcessInfo(), + this.Capture, this.Ocr, new Snapshot(new()), this.Tracker, this.recorder, + this.Translator, new Cache(), this.Color, delayFilter ? [this.Filter] : [], NullLogger.Instance); + } + finally + { + if (withoutContext) SynchronizationContext.SetSynchronizationContext(context); + } + this.submitCapture = typeof(MainViewModelBase).GetMethod("QueueCapture", BindingFlags.Instance | BindingFlags.NonPublic)! + .CreateDelegate>(this.Model); + } + + public Capture Capture { get; } = new(); + public Ocr Ocr { get; } + public Tracker Tracker { get; } + public Color Color { get; } + public Translator Translator { get; } + public Filter Filter { get; } + public OverlayMainViewModel Model { get; } + public System.Windows.Threading.DispatcherTimer TrackingTimer => Field("trackingTimer"); + public Subject<(IEnumerable Texts, FilterContext Context)> Tracking + => Field, FilterContext)>>("trackingRequests"); + public Subject<(IEnumerable Texts, FilterContext Context)?> Translations + => Field, FilterContext)?>>("translationRequests"); + + public void SubmitCapture(SoftwareBitmap bitmap) => this.submitCapture(bitmap); + + private T Field(string name) where T : class + => (T)typeof(MainViewModelBase).GetField(name, BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(this.Model)!; + + public async ValueTask DisposeAsync() + { + this.Model.Dispose(); + await this.recorder.DisposeAsync(); + this.Tracker.ResetRelease.Dispose(); + this.Tracker.UpdateRelease.Dispose(); + } + } + + private sealed class Gate(bool delayed) + { + private int count; + public int Count => Volatile.Read(ref this.count); + public TaskCompletionSource Started { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource NextStarted { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource Release { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public async Task EnterAsync() + { + var current = Interlocked.Increment(ref this.count); + if (current == 1) this.Started.TrySetResult(); + else this.NextStarted.TrySetResult(); + if (delayed && current == 1) await this.Release.Task.WaitAsync(TimeSpan.FromSeconds(10)).ConfigureAwait(false); + } + } + + private sealed class Snapshot(T value) : IOptionsSnapshot where T : class + { + public T Value { get; } = value; + public T Get(string? name) => this.Value; + } + + private sealed class ProcessInfo : IProcessInfoStore + { + public IntPtr MainWindowHandle => IntPtr.Zero; + public string Name => "test"; + } + + private sealed class Capture : ICaptureModule + { + public event AsyncEventHandler? Captured; + public AsyncEventHandler? CurrentHandler => this.Captured; + public Action? OnStopped { get; set; } + public int StartCount { get; private set; } + public void StartCapture(IntPtr targetWindow) => this.StartCount++; + public void StopCapture() => this.OnStopped?.Invoke(); + } + + private sealed class Ocr(bool delayed, bool multipleTexts, bool failFirst) : IOcrModule + { + public Gate Gate { get; } = new(delayed); + public async ValueTask> RecognizeAsync(OcrCaptureInput input) + { + await this.Gate.EnterAsync(); + if (failFirst && this.Gate.Count == 1) throw new InvalidOperationException("模擬 OCR エラー"); + var text = input.Source.PixelWidth == 321 ? "new" : "old"; + var first = new TextRect(text, 0, 0, 100, 30, 20, false); + return multipleTexts ? [first, first with { SourceText = "old2", Y = 40 }] : [first]; + } + } + + private sealed class Tracker(bool delayReset, bool delayUpdate) : IOcrTextTracker + { + private int updates; + private int resets; + private TextRect[] previous = []; + public TaskCompletionSource ResetStarted { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public ManualResetEventSlim ResetRelease { get; } = new(); + public ManualResetEventSlim UpdateRelease { get; } = new(); + public TaskCompletionSource Updated { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public int UpdateCount => Volatile.Read(ref this.updates); + public int ResetCount => Volatile.Read(ref this.resets); + public IReadOnlyList Update(IEnumerable observations, System.Drawing.Size imageSize, TimeSpan? timestamp = null) + { + var count = Interlocked.Increment(ref this.updates); + this.Updated.TrySetResult(); + if (delayUpdate && count == 1 && !this.UpdateRelease.Wait(TimeSpan.FromSeconds(10))) + { + throw new TimeoutException(); + } + var current = observations.ToArray(); + this.previous = [.. this.previous.Where(old => current.All(text => text.SourceText != old.SourceText)), .. current]; + return this.previous; + } + public void Reset() + { + var count = Interlocked.Increment(ref this.resets); + this.previous = []; + this.ResetStarted.TrySetResult(); + if (delayReset && count == 1 && !this.ResetRelease.Wait(TimeSpan.FromSeconds(10))) + { + throw new TimeoutException(); + } + } + } + + private sealed class Color(bool delayed) : IColorModule + { + public Gate Gate { get; } = new(delayed); + public async ValueTask> ConvertColorAsync(SoftwareBitmap bitmap, IEnumerable texts) + { + var result = texts.ToArray(); + await this.Gate.EnterAsync(); + return result; + } + } + + private sealed class Translator(bool delayed, bool failFirst) : ITranslateModule + { + public Gate Gate { get; } = new(delayed); + public async ValueTask TranslateAsync(TextInfo[] srcTexts) + { + await this.Gate.EnterAsync(); + if (failFirst && this.Gate.Count == 1) throw new InvalidOperationException("模擬翻訳エラー"); + return srcTexts.Select(text => text.SourceText).ToArray(); + } + } + + private sealed class Filter(bool delayed) : IFilterModule + { + public Gate Gate { get; } = new(delayed); + public async IAsyncEnumerable ExecutePreTranslate(IAsyncEnumerable texts, FilterContext context) + { + await this.Gate.EnterAsync(); + await foreach (var text in texts) yield return text; + } + + public IAsyncEnumerable ExecutePostTranslate(IAsyncEnumerable texts, FilterContext context) => texts; + } + + private sealed class Cache : ICacheModule + { + public bool Contains(string src) => false; + public string Get(string src) => throw new InvalidOperationException(); + public void AddRange(IEnumerable<(string src, string dst)> pairs) { } + } +} diff --git a/WindowTranslator/Modules/Main/MainViewModelBase.cs b/WindowTranslator/Modules/Main/MainViewModelBase.cs index a2ecbe6d..324afae2 100644 --- a/WindowTranslator/Modules/Main/MainViewModelBase.cs +++ b/WindowTranslator/Modules/Main/MainViewModelBase.cs @@ -8,6 +8,7 @@ using Kamishibai; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; +using Microsoft.VisualStudio.Threading; using R3; using Windows.Graphics.Imaging; using WindowTranslator.ComponentModel; @@ -22,14 +23,17 @@ namespace WindowTranslator.Modules.Main; [ObservableObject] public abstract partial class MainViewModelBase : IDisposable { + private readonly Dispatcher dispatcher = Application.Current.Dispatcher; private readonly Subject captureRequests = new(); - private readonly Subject trackingRequests = new(); - private readonly Subject<(IEnumerable Texts, FilterContext Context)> translationRequests = new(); - private readonly Subject> uiRequests = new(); + private readonly Subject recognitionRequests = new(); + private readonly Subject<(IEnumerable Texts, FilterContext Context)> trackingRequests = new(); + private readonly Subject<(IEnumerable Texts, FilterContext Context)?> translationRequests = new(); + private IDisposable captureEvents; + private readonly IDisposable captureInput; private readonly IDisposable captureLoop; private readonly IDisposable trackingLoop; private readonly IDisposable translationLoop; - private readonly IDisposable uiLoop; + private readonly DispatcherTimer trackingTimer; private readonly IOcrModule ocr; private readonly List priorityRects; private readonly IOcrTextTracker ocrTextTracker; @@ -47,6 +51,7 @@ public abstract partial class MainViewModelBase : IDisposable private readonly double overlayOpacity; private readonly double mousePointerHitTestPadding; private readonly bool isOneShotMode; + private readonly TimeSpan captureInterval; [ObservableProperty] private string title; @@ -65,7 +70,11 @@ public abstract partial class MainViewModelBase : IDisposable public BusyScope Filtering { get; } = new(); private bool isFirstCapture = true; + private bool needsTrackerReset; + private bool updatingOcrTexts; + private SoftwareBitmap? pendingCapture; private volatile bool disposedValue; + private event Action? OverlayHidden; public ObservableCollection OcrTexts { get; } = []; public string Font { get; } @@ -94,9 +103,10 @@ public MainViewModelBase( this.overlayOpacity = options.Value.OverlayOpacity; this.mousePointerHitTestPadding = options.Value.MousePointerHitTestPadding; this.isOneShotMode = options.Value.IsOneShotMode; + this.captureInterval = TimeSpan.FromSeconds(options.Value.CaptureInterval); this.DisplayBusy = options.Value.DisplayBusy; this.capture = capture ?? throw new ArgumentNullException(nameof(capture)); - this.capture.Captured += Capture_CapturedAsync; + this.captureEvents = CreateCaptureEvents(); this.ocr = ocr ?? throw new ArgumentNullException(nameof(ocr)); this.priorityRects = ocrParam.Value.PriorityRects ?? []; this.ocrTextTracker = ocrTextTracker ?? throw new ArgumentNullException(nameof(ocrTextTracker)); @@ -106,80 +116,98 @@ public MainViewModelBase( this.color = color ?? throw new ArgumentNullException(nameof(color)); this.filters = filters.ToArray(); this.logger = logger; - this.uiLoop = this.uiRequests - .ObserveOn(new DispatcherSynchronizationContext(Application.Current.Dispatcher)) - .Where(_ => this.OverlayVisible) - .Subscribe(UpdateOcrTexts); - var captureInterval = TimeSpan.FromSeconds(options.Value.CaptureInterval); - var captures = !this.isOneShotMode && captureInterval > TimeSpan.Zero - ? this.captureRequests.ThrottleLast(captureInterval) + // 処理開始と非表示通知を UI スレッドで順序付ける。 + this.captureLoop = this.dispatcher.Invoke(() => this.recognitionRequests + .SubscribeAwaitThrottleFirstLast( + (bitmap, ct) => bitmap is null ? ValueTask.CompletedTask : RecognizeAsync(bitmap, ct), + static bitmap => bitmap?.Dispose(), configureAwait: true)); + var captures = !this.isOneShotMode && this.captureInterval > TimeSpan.Zero + ? this.captureRequests.ThrottleLast(this.captureInterval) : this.captureRequests; - this.captureLoop = captures - .ObserveOnThreadPool() - .SubscribeAwait(RecognizeAsync, AwaitOperation.ThrottleFirstLast, configureAwait: false); + this.captureInput = captures.Subscribe(bitmap => + { + _ = this.dispatcher.InvokeAsync(() => + { + if (ReferenceEquals(Interlocked.CompareExchange(ref this.pendingCapture, null, bitmap), bitmap)) + { + this.recognitionRequests.OnNext(bitmap); + } + }); + }); + this.trackingTimer = new DispatcherTimer(DispatcherPriority.Normal, this.dispatcher) + { + Interval = TimeSpan.FromMilliseconds(100), + }; + this.trackingTimer.Tick += (_, _) => this.trackingRequests.OnNext(default); this.trackingLoop = Disposable.Combine( Disposable.Create(() => this.trackingRequests.OnCompleted()), - Observable.Timer(TimeSpan.Zero, TimeSpan.FromMilliseconds(100), TimeProvider.System) - .Where(_ => this.OverlayVisible) - .Subscribe(_ => this.trackingRequests.OnNext(default)), this.trackingRequests .Materialize() .Synchronize() - .Scan(default(TrackingRequest), (previous, current) => + // タイマーの空通知では直前のOCR結果を再利用する。 + .Scan(default((IEnumerable Texts, FilterContext Context)), (previous, current) => { - if (current.Kind != NotificationKind.OnNext) - { - previous.Context?.SoftwareBitmap.Dispose(); - return default; - } - - if (current.Value.Reset) - { - previous.Context?.SoftwareBitmap.Dispose(); - this.ocrTextTracker.Reset(); - return default; - } - if (!this.OverlayVisible) + if (!this.OverlayVisible && current.Kind == NotificationKind.OnNext) { previous.Context?.SoftwareBitmap.Dispose(); current.Value.Context?.SoftwareBitmap.Dispose(); return default; } - if (current.Value.Context is null) + if (current.Value.Context is { } context) { - return previous; + if (previous.Context?.ImageSize != context.ImageSize) + { + this.needsTrackerReset = true; + } + this.isFirstCapture |= this.isOneShotMode; } - if (previous.Context?.ImageSize != current.Value.Context.ImageSize) + else if (current.Kind == NotificationKind.OnNext) { - this.ocrTextTracker.Reset(); + return previous; } previous.Context?.SoftwareBitmap.Dispose(); - this.isFirstCapture |= this.isOneShotMode; return current.Value; }) .Where(request => request.Context is not null) .SubscribeAwait(async (request, ct) => { + var hidden = new TaskCompletionSource(); + using var visibility = ObserveHiding(hidden); if (!this.OverlayVisible) { return; } - var context = request.Context!; - using var bitmap = context.SoftwareBitmap.GetReadOnlyView(); - var texts = await ProcessTrackingAsync((request.Texts, context), bitmap, ct); - if (!this.OverlayVisible) + using var bitmap = request.Context.SoftwareBitmap.GetReadOnlyView(); + var reset = this.needsTrackerReset; + this.needsTrackerReset = false; + // 重い追跡処理は UI スレッドを離れて実行する。 + await Task.CompletedTask.ConfigureAwait(ConfigureAwaitOptions.ForceYielding); + ct.ThrowIfCancellationRequested(); + if (hidden.Task.IsCompleted || !this.OverlayVisible) return; + if (reset) this.ocrTextTracker.Reset(); + if (hidden.Task.IsCompleted || !this.OverlayVisible) return; + var texts = await ProcessTrackingAsync(request, bitmap, hidden.Task, ct); + if (texts is null) { return; } - if (this.isFirstCapture) + await this.dispatcher.InvokeAsync(() => { - this.isFirstCapture = false; - this.uiRequests.OnNext(texts); - } - this.translationRequests.OnNext((texts, context with { SoftwareBitmap = bitmap.GetReadOnlyView(), })); + if (hidden.Task.IsCompleted || this.disposedValue || !this.OverlayVisible) return; + if (this.isFirstCapture) + { + this.isFirstCapture = false; + UpdateOcrTexts(texts, hidden.Task); + } + if (hidden.Task.IsCompleted || !this.OverlayVisible) return; + this.translationRequests.OnNext((texts, request.Context with { SoftwareBitmap = bitmap.GetReadOnlyView(), })); + }); }, AwaitOperation.Drop, configureAwait: false, cancelOnCompleted: true)); - this.translationLoop = this.translationRequests - .SubscribeAwaitThrottleFirstLast(ProcessTranslationAsync, static r => r.Context.SoftwareBitmap.Dispose(), configureAwait: false); + this.translationLoop = this.dispatcher.Invoke(() => this.translationRequests + .SubscribeAwaitThrottleFirstLast( + (request, ct) => request is { } value ? ProcessTranslationAsync(value, ct) : ValueTask.CompletedTask, + static request => request?.Context.SoftwareBitmap.Dispose(), configureAwait: true)); + this.trackingTimer.Start(); if (!this.isOneShotMode) { this.capture.StartCapture(this.processInfoStore.MainWindowHandle); @@ -188,46 +216,113 @@ public MainViewModelBase( this.title = $"{this.name} - {this.translator.Name} ({transAsm.GetName().Version})"; } + private IDisposable ObserveHiding(TaskCompletionSource hidden) + { + void Handler() => hidden.TrySetResult(); + this.OverlayHidden += Handler; + return Disposable.Create(() => this.OverlayHidden -= Handler); + } + + private void QueueCapture(SoftwareBitmap bitmap) + { + if (this.isOneShotMode || this.captureInterval == TimeSpan.Zero) + { + this.recognitionRequests.OnNext(bitmap); + return; + } + Interlocked.Exchange(ref this.pendingCapture, bitmap)?.Dispose(); + this.captureRequests.OnNext(bitmap); + } + + private void ClearPendingCapture() + => Interlocked.Exchange(ref this.pendingCapture, null)?.Dispose(); + partial void OnOverlayVisibleChanged(bool value) { - this.trackingRequests.OnNext(new([], null, Reset: true)); if (value) { - this.uiRequests.OnNext([]); this.isFirstCapture = !this.isOneShotMode; - // Start capture when overlay becomes visible - this.capture.StartCapture(this.processInfoStore.MainWindowHandle); + this.captureEvents = CreateCaptureEvents(); + if (this.updatingOcrTexts) + { + _ = this.dispatcher.InvokeAsync(this.OcrTexts.Clear, DispatcherPriority.Send); + } + else if (this.OcrTexts.Count > 0) + { + this.OcrTexts.Clear(); + } + if (this.OverlayVisible && !this.trackingTimer.IsEnabled) + { + this.trackingTimer.Start(); + // Start capture when overlay becomes visible + this.capture.StartCapture(this.processInfoStore.MainWindowHandle); + } } else { + this.OverlayHidden?.Invoke(); + this.captureEvents.Dispose(); + ClearPendingCapture(); + this.recognitionRequests.OnNext(null); + this.translationRequests.OnNext(null); + this.trackingTimer.Stop(); + this.trackingRequests.OnNext(default); // Stop capture when overlay becomes hidden this.capture.StopCapture(); this.logger.LogDebug("Overlay became hidden - capture stopped"); } } - private async Task Capture_CapturedAsync(object? sender, CapturedEventArgs args) + private IDisposable CreateCaptureEvents() + => Observable.Create(observer => + { + AsyncEventHandler handler = (_, args) => Capture_CapturedAsync(args, observer); + this.capture.Captured += handler; + return Disposable.Create(() => this.capture.Captured -= handler); + }, rawObserver: true).Subscribe(QueueCapture); + + private async Task Capture_CapturedAsync(CapturedEventArgs args, Observer observer) { - if (this.isOneShotMode) + var accepted = await this.dispatcher.InvokeAsync(() => { - this.capture.StopCapture(); - } + if (this.disposedValue || !this.OverlayVisible || observer.IsDisposed) return false; + if (this.isOneShotMode) this.capture.StopCapture(); + return true; + }); + if (!accepted) return; // フレームの寿命はイベント処理中だけなので、コピーの完了まで待つ。 var bitmap = await SoftwareBitmap.CreateCopyFromSurfaceAsync(args.Frame.Surface); - if (this.disposedValue) + var handedOff = false; + try { - bitmap.Dispose(); - return; + await this.dispatcher.InvokeAsync(() => + { + if (this.disposedValue || !this.OverlayVisible || observer.IsDisposed) return; + this.Width = bitmap.PixelWidth; + this.Height = bitmap.PixelHeight; + if (this.disposedValue || !this.OverlayVisible || observer.IsDisposed) return; + observer.OnNext(bitmap); + handedOff = true; + }); + } + finally + { + if (!handedOff) bitmap.Dispose(); } - this.Width = bitmap.PixelWidth; - this.Height = bitmap.PixelHeight; - this.captureRequests.OnNext(bitmap); } private async ValueTask RecognizeAsync(SoftwareBitmap capturedBitmap, CancellationToken cancellationToken) { using var bitmap = new DisposeAction(capturedBitmap.Dispose); + var hidden = new TaskCompletionSource(); + using var visibility = ObserveHiding(hidden); + await Task.CompletedTask.ConfigureAwait(ConfigureAwaitOptions.ForceYielding); + cancellationToken.ThrowIfCancellationRequested(); + if (this.disposedValue || hidden.Task.IsCompleted || !this.OverlayVisible) + { + return; + } IEnumerable texts; using (this.Recognizing.EnterBusy()) { @@ -270,21 +365,22 @@ private async ValueTask RecognizeAsync(SoftwareBitmap capturedBitmap, Cancellati catch (ObjectDisposedException) { // すでに破棄されている場合は何もしない - this.captureLoop.Dispose(); - await Application.Current.Dispatcher.InvokeAsync(this.capture.StopCapture); + await StopRecognitionAsync(hidden.Task); return; } catch (OperationCanceledException) { // キャンセルされた場合は何もしない - this.captureLoop.Dispose(); - await Application.Current.Dispatcher.InvokeAsync(this.capture.StopCapture); + await StopRecognitionAsync(hidden.Task); return; } catch (Exception e) { - this.captureLoop.Dispose(); - await Application.Current.Dispatcher.InvokeAsync(this.capture.StopCapture); + if (!await StopRecognitionAsync(hidden.Task)) + { + this.logger.LogError(e, "非表示前の画像認識でエラーが発生"); + return; + } var path = Path.Combine(PathUtility.UserDir, $"ocr_error", $"{DateTime.UtcNow:yyyyMMdd'T'HHmmss'Z'}.png"); await capturedBitmap.TrySaveImage(path); await ShowErrorAsync(Resources.FaildOcr, e, path); @@ -292,28 +388,52 @@ private async ValueTask RecognizeAsync(SoftwareBitmap capturedBitmap, Cancellati } } cancellationToken.ThrowIfCancellationRequested(); - if (this.disposedValue) - { - return; - } - var context = new FilterContext + await this.dispatcher.InvokeAsync(() => { - SoftwareBitmap = capturedBitmap.GetReadOnlyView(), - ImageSize = new(capturedBitmap.PixelWidth, capturedBitmap.PixelHeight), - }; - this.trackingRequests.OnNext(new(texts, context)); + if (this.disposedValue || hidden.Task.IsCompleted || !this.OverlayVisible) return; + var context = new FilterContext + { + SoftwareBitmap = capturedBitmap.GetReadOnlyView(), + ImageSize = new(capturedBitmap.PixelWidth, capturedBitmap.PixelHeight), + }; + this.trackingRequests.OnNext((texts, context)); + }); } + private async Task StopRecognitionAsync(Task hidden) + => await this.dispatcher.InvokeAsync(() => + { + if (this.disposedValue || hidden.IsCompleted || !this.OverlayVisible) return false; + this.captureEvents.Dispose(); + ClearPendingCapture(); + this.captureInput.Dispose(); + this.captureLoop.Dispose(); + this.capture.StopCapture(); + return true; + }); + internal static IEnumerable SelectOcrTexts( IEnumerable observations, System.Drawing.Size imageSize, IOcrTextTracker tracker, bool isOneShotMode) => isOneShotMode ? observations : tracker.Update(observations, imageSize); - private async ValueTask> ProcessTrackingAsync((IEnumerable Texts, FilterContext Context) request, SoftwareBitmap bitmap, CancellationToken cancellationToken) + private async ValueTask?> ProcessTrackingAsync( + (IEnumerable Texts, FilterContext Context) request, + SoftwareBitmap bitmap, + Task hidden, + CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); var texts = SelectOcrTexts(request.Texts, request.Context.ImageSize, this.ocrTextTracker, this.isOneShotMode) .Select(t => t with { FontSize = t.FontSize * this.fontScale }); + if (hidden.IsCompleted || !this.OverlayVisible) + { + return null; + } texts = (await this.color.ConvertColorAsync(bitmap, texts)).ToArray(); + if (hidden.IsCompleted || !this.OverlayVisible) + { + return null; + } texts = texts.Select(t => t switch { { TranslatedText: null } when this.cache.Contains(t.SourceText) => t with { TranslatedText = this.cache.Get(t.SourceText) }, @@ -326,8 +446,11 @@ private async ValueTask> ProcessTrackingAsync((IEnumerable private async ValueTask ProcessTranslationAsync((IEnumerable Texts, FilterContext Context) request, CancellationToken cancellationToken) { using var bitmap = request.Context.SoftwareBitmap; + var hidden = new TaskCompletionSource(); + using var visibility = ObserveHiding(hidden); + await Task.CompletedTask.ConfigureAwait(ConfigureAwaitOptions.ForceYielding); cancellationToken.ThrowIfCancellationRequested(); - if (this.disposedValue || !this.OverlayVisible) + if (this.disposedValue || hidden.Task.IsCompleted || !this.OverlayVisible) { return; } @@ -342,7 +465,7 @@ private async ValueTask ProcessTranslationAsync((IEnumerable Texts, Fi using var t = this.logger.LogDebugTime("翻訳前処理"); texts = await tmp.ToArrayAsync(cancellationToken); } - if (!this.OverlayVisible) + if (hidden.Task.IsCompleted || !this.OverlayVisible) { return; } @@ -354,11 +477,8 @@ private async ValueTask ProcessTranslationAsync((IEnumerable Texts, Fi if (requests.Length > 0) { using var t = this.logger.LogDebugTime("翻訳"); + if (hidden.Task.IsCompleted || !this.OverlayVisible) return; var translated = await this.translator.TranslateAsync(requests).ConfigureAwait(false); - if (!this.OverlayVisible) - { - return; - } this.cache.AddRange(requests.Select(t => t.SourceText).Zip(translated)); var translatedIndex = 0; texts = texts.Select(t => t.TranslatedText is null @@ -377,11 +497,20 @@ private async ValueTask ProcessTranslationAsync((IEnumerable Texts, Fi catch (Exception e) { this.logger.LogError(e, "翻訳中にエラーが発生"); - this.captureLoop.Dispose(); - this.trackingLoop.Dispose(); - await Application.Current.Dispatcher.InvokeAsync(this.capture.StopCapture); - this.translationLoop.Dispose(); - await ShowErrorAsync(Resources.FaildOverlay, e, string.Empty); + var stopped = await this.dispatcher.InvokeAsync(() => + { + if (hidden.Task.IsCompleted || this.disposedValue || !this.OverlayVisible) return false; + this.captureEvents.Dispose(); + ClearPendingCapture(); + this.captureInput.Dispose(); + this.captureLoop.Dispose(); + this.trackingTimer.Stop(); + this.trackingLoop.Dispose(); + this.translationLoop.Dispose(); + this.capture.StopCapture(); + return true; + }); + if (stopped) await ShowErrorAsync(Resources.FaildOverlay, e, string.Empty); return; } using (this.Filtering.EnterBusy()) @@ -395,31 +524,42 @@ private async ValueTask ProcessTranslationAsync((IEnumerable Texts, Fi texts = await tmp.ToArrayAsync(cancellationToken); } cancellationToken.ThrowIfCancellationRequested(); - if (this.OverlayVisible) + await this.dispatcher.InvokeAsync(() => { - this.uiRequests.OnNext(texts); - } + if (!hidden.Task.IsCompleted && !this.disposedValue && this.OverlayVisible) + { + UpdateOcrTexts(texts, hidden.Task); + } + }); } - private void UpdateOcrTexts(IEnumerable texts) + private void UpdateOcrTexts(IEnumerable texts, Task hidden) { this.logger.LogDebug("UI適用"); var hash = texts.Select(t => t with { Background = Color.FromArgb((int)(255 * this.overlayOpacity), t.Background) }).ToHashSet(); - foreach (var text in this.OcrTexts.Where(t => !hash.Contains(t)).ToArray()) + this.updatingOcrTexts = true; + try { - this.OcrTexts.Remove(text); + foreach (var text in this.OcrTexts.Where(t => !hash.Contains(t)).ToArray()) + { + if (hidden.IsCompleted) return; + this.OcrTexts.Remove(text); + } + hash.ExceptWith(this.OcrTexts); + foreach (var text in hash) + { + if (hidden.IsCompleted) return; + this.OcrTexts.Add(text); + } } - hash.ExceptWith(this.OcrTexts); - foreach (var text in hash) + finally { - this.OcrTexts.Add(text); + this.updatingOcrTexts = false; } } - private readonly record struct TrackingRequest(IEnumerable Texts, FilterContext? Context, bool Reset = false); - private Task ShowErrorAsync(string message, Exception exception, string path) - => Application.Current.Dispatcher.InvokeAsync(async () => + => this.dispatcher.InvokeAsync(async () => { await this.presentationService.OpenErrorDialogAsync(message, exception, this.name, path); StrongReferenceMessenger.Default.Send(new(this)); @@ -435,17 +575,20 @@ protected virtual void Dispose(bool disposing) if (disposing) { // 実行中の処理から遅れて通知されても受け付けない。 + this.captureEvents.Dispose(); + ClearPendingCapture(); this.captureRequests.OnCompleted(); + this.recognitionRequests.OnCompleted(); this.translationRequests.OnCompleted(); - this.uiRequests.OnCompleted(); if (this.capture is IDisposable captureDisposable) { captureDisposable.Dispose(); } + this.captureInput.Dispose(); this.captureLoop.Dispose(); + this.trackingTimer.Stop(); this.trackingLoop.Dispose(); this.translationLoop.Dispose(); - this.uiLoop.Dispose(); } } From e9eb5a90634bcfe215cfc566be9c7f1bd33e545b Mon Sep 17 00:00:00 2001 From: Freesia Date: Tue, 3 Nov 2026 22:37:28 +0900 Subject: [PATCH 5/8] =?UTF-8?q?=E9=9D=9E=E8=A1=A8=E7=A4=BA=E6=99=82?= =?UTF-8?q?=E3=81=AE=E9=87=8D=E8=A4=87=E5=88=A4=E5=AE=9A=E3=82=92=E9=99=A4?= =?UTF-8?q?=E5=8E=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../OverlayVisibilityTests.cs | 14 ++++- .../Modules/Main/MainViewModelBase.cs | 57 ++++++++++--------- 2 files changed, 41 insertions(+), 30 deletions(-) diff --git a/WindowTranslator.Tests/OverlayVisibilityTests.cs b/WindowTranslator.Tests/OverlayVisibilityTests.cs index 7a1e9f65..2bb9b11b 100644 --- a/WindowTranslator.Tests/OverlayVisibilityTests.cs +++ b/WindowTranslator.Tests/OverlayVisibilityTests.cs @@ -91,6 +91,7 @@ private static async Task RunInStaAsync() await VerifyReentrantHideAsync(); await VerifyNestedVisibilityAsync(); await VerifyPendingAsync(); + await VerifyOneShotAsync(); await VerifyDelayedCaptureAsync(); await VerifyDisposeAsync(); afterShutdown = new Fixture(captureInterval: 0.5); @@ -332,6 +333,15 @@ private static async Task VerifyDelayedCaptureAsync() Assert.False(fixture.Ocr.Gate.Started.Task.IsCompleted); } + private static async Task VerifyOneShotAsync() + { + await using var fixture = new Fixture(captureInterval: 10, isOneShotMode: true); + fixture.SubmitCapture(Bitmap(321)); + await fixture.Ocr.Gate.Started.Task.WaitAsync(TimeSpan.FromSeconds(5)); + await WaitUntilAsync(() => fixture.Model.OcrTexts.Any(text => text.SourceText == "new")); + Assert.Equal(0, fixture.Tracker.UpdateCount); + } + private static async Task VerifyReentrantShowAsync() { await using var fixture = new Fixture(Stage.Ocr); @@ -444,7 +454,7 @@ private sealed class Fixture : System.IAsyncDisposable private readonly OcrTraceRecorder recorder = new(NullLogger.Instance); private readonly Action submitCapture; - public Fixture(Stage? delayed = null, double captureInterval = 0, bool delayReset = false, bool delayFilter = false, bool multipleTexts = false, bool withoutContext = false, bool failFirst = false, bool delayUpdate = false) + public Fixture(Stage? delayed = null, double captureInterval = 0, bool delayReset = false, bool delayFilter = false, bool multipleTexts = false, bool withoutContext = false, bool failFirst = false, bool delayUpdate = false, bool isOneShotMode = false) { this.Ocr = new(delayed == Stage.Ocr, multipleTexts, failFirst && delayed == Stage.Ocr); this.Color = new(delayed == Stage.Color); @@ -456,7 +466,7 @@ public Fixture(Stage? delayed = null, double captureInterval = 0, bool delayRese { if (withoutContext) SynchronizationContext.SetSynchronizationContext(null); this.Model = new OverlayMainViewModel( - null!, new Snapshot(new() { CaptureInterval = captureInterval }), new ProcessInfo(), + null!, new Snapshot(new() { CaptureInterval = captureInterval, IsOneShotMode = isOneShotMode }), new ProcessInfo(), this.Capture, this.Ocr, new Snapshot(new()), this.Tracker, this.recorder, this.Translator, new Cache(), this.Color, delayFilter ? [this.Filter] : [], NullLogger.Instance); } diff --git a/WindowTranslator/Modules/Main/MainViewModelBase.cs b/WindowTranslator/Modules/Main/MainViewModelBase.cs index 324afae2..61180a6e 100644 --- a/WindowTranslator/Modules/Main/MainViewModelBase.cs +++ b/WindowTranslator/Modules/Main/MainViewModelBase.cs @@ -121,19 +121,18 @@ public MainViewModelBase( .SubscribeAwaitThrottleFirstLast( (bitmap, ct) => bitmap is null ? ValueTask.CompletedTask : RecognizeAsync(bitmap, ct), static bitmap => bitmap?.Dispose(), configureAwait: true)); - var captures = !this.isOneShotMode && this.captureInterval > TimeSpan.Zero - ? this.captureRequests.ThrottleLast(this.captureInterval) - : this.captureRequests; - this.captureInput = captures.Subscribe(bitmap => - { - _ = this.dispatcher.InvokeAsync(() => + this.captureInput = !this.isOneShotMode && this.captureInterval > TimeSpan.Zero + ? this.captureRequests.ThrottleLast(this.captureInterval).Subscribe(bitmap => { - if (ReferenceEquals(Interlocked.CompareExchange(ref this.pendingCapture, null, bitmap), bitmap)) + _ = this.dispatcher.InvokeAsync(() => { - this.recognitionRequests.OnNext(bitmap); - } - }); - }); + if (ReferenceEquals(Interlocked.CompareExchange(ref this.pendingCapture, null, bitmap), bitmap)) + { + this.recognitionRequests.OnNext(bitmap); + } + }); + }) + : Disposable.Empty; this.trackingTimer = new DispatcherTimer(DispatcherPriority.Normal, this.dispatcher) { Interval = TimeSpan.FromMilliseconds(100), @@ -183,9 +182,9 @@ public MainViewModelBase( // 重い追跡処理は UI スレッドを離れて実行する。 await Task.CompletedTask.ConfigureAwait(ConfigureAwaitOptions.ForceYielding); ct.ThrowIfCancellationRequested(); - if (hidden.Task.IsCompleted || !this.OverlayVisible) return; + if (hidden.Task.IsCompleted) return; if (reset) this.ocrTextTracker.Reset(); - if (hidden.Task.IsCompleted || !this.OverlayVisible) return; + if (hidden.Task.IsCompleted) return; var texts = await ProcessTrackingAsync(request, bitmap, hidden.Task, ct); if (texts is null) { @@ -193,13 +192,13 @@ public MainViewModelBase( } await this.dispatcher.InvokeAsync(() => { - if (hidden.Task.IsCompleted || this.disposedValue || !this.OverlayVisible) return; + if (hidden.Task.IsCompleted || this.disposedValue) return; if (this.isFirstCapture) { this.isFirstCapture = false; UpdateOcrTexts(texts, hidden.Task); } - if (hidden.Task.IsCompleted || !this.OverlayVisible) return; + if (hidden.Task.IsCompleted) return; this.translationRequests.OnNext((texts, request.Context with { SoftwareBitmap = bitmap.GetReadOnlyView(), })); }); }, AwaitOperation.Drop, configureAwait: false, cancelOnCompleted: true)); @@ -285,7 +284,7 @@ private async Task Capture_CapturedAsync(CapturedEventArgs args, Observer { - if (this.disposedValue || !this.OverlayVisible || observer.IsDisposed) return false; + if (this.disposedValue || observer.IsDisposed) return false; if (this.isOneShotMode) this.capture.StopCapture(); return true; }); @@ -298,10 +297,10 @@ private async Task Capture_CapturedAsync(CapturedEventArgs args, Observer { - if (this.disposedValue || !this.OverlayVisible || observer.IsDisposed) return; + if (this.disposedValue || observer.IsDisposed) return; this.Width = bitmap.PixelWidth; this.Height = bitmap.PixelHeight; - if (this.disposedValue || !this.OverlayVisible || observer.IsDisposed) return; + if (this.disposedValue || observer.IsDisposed) return; observer.OnNext(bitmap); handedOff = true; }); @@ -317,9 +316,10 @@ private async ValueTask RecognizeAsync(SoftwareBitmap capturedBitmap, Cancellati using var bitmap = new DisposeAction(capturedBitmap.Dispose); var hidden = new TaskCompletionSource(); using var visibility = ObserveHiding(hidden); + if (!this.OverlayVisible) return; await Task.CompletedTask.ConfigureAwait(ConfigureAwaitOptions.ForceYielding); cancellationToken.ThrowIfCancellationRequested(); - if (this.disposedValue || hidden.Task.IsCompleted || !this.OverlayVisible) + if (this.disposedValue || hidden.Task.IsCompleted) { return; } @@ -390,7 +390,7 @@ private async ValueTask RecognizeAsync(SoftwareBitmap capturedBitmap, Cancellati cancellationToken.ThrowIfCancellationRequested(); await this.dispatcher.InvokeAsync(() => { - if (this.disposedValue || hidden.Task.IsCompleted || !this.OverlayVisible) return; + if (this.disposedValue || hidden.Task.IsCompleted) return; var context = new FilterContext { SoftwareBitmap = capturedBitmap.GetReadOnlyView(), @@ -403,7 +403,7 @@ await this.dispatcher.InvokeAsync(() => private async Task StopRecognitionAsync(Task hidden) => await this.dispatcher.InvokeAsync(() => { - if (this.disposedValue || hidden.IsCompleted || !this.OverlayVisible) return false; + if (this.disposedValue || hidden.IsCompleted) return false; this.captureEvents.Dispose(); ClearPendingCapture(); this.captureInput.Dispose(); @@ -425,12 +425,12 @@ internal static IEnumerable SelectOcrTexts( cancellationToken.ThrowIfCancellationRequested(); var texts = SelectOcrTexts(request.Texts, request.Context.ImageSize, this.ocrTextTracker, this.isOneShotMode) .Select(t => t with { FontSize = t.FontSize * this.fontScale }); - if (hidden.IsCompleted || !this.OverlayVisible) + if (hidden.IsCompleted) { return null; } texts = (await this.color.ConvertColorAsync(bitmap, texts)).ToArray(); - if (hidden.IsCompleted || !this.OverlayVisible) + if (hidden.IsCompleted) { return null; } @@ -448,9 +448,10 @@ private async ValueTask ProcessTranslationAsync((IEnumerable Texts, Fi using var bitmap = request.Context.SoftwareBitmap; var hidden = new TaskCompletionSource(); using var visibility = ObserveHiding(hidden); + if (!this.OverlayVisible) return; await Task.CompletedTask.ConfigureAwait(ConfigureAwaitOptions.ForceYielding); cancellationToken.ThrowIfCancellationRequested(); - if (this.disposedValue || hidden.Task.IsCompleted || !this.OverlayVisible) + if (this.disposedValue || hidden.Task.IsCompleted) { return; } @@ -465,7 +466,7 @@ private async ValueTask ProcessTranslationAsync((IEnumerable Texts, Fi using var t = this.logger.LogDebugTime("翻訳前処理"); texts = await tmp.ToArrayAsync(cancellationToken); } - if (hidden.Task.IsCompleted || !this.OverlayVisible) + if (hidden.Task.IsCompleted) { return; } @@ -477,7 +478,7 @@ private async ValueTask ProcessTranslationAsync((IEnumerable Texts, Fi if (requests.Length > 0) { using var t = this.logger.LogDebugTime("翻訳"); - if (hidden.Task.IsCompleted || !this.OverlayVisible) return; + if (hidden.Task.IsCompleted) return; var translated = await this.translator.TranslateAsync(requests).ConfigureAwait(false); this.cache.AddRange(requests.Select(t => t.SourceText).Zip(translated)); var translatedIndex = 0; @@ -499,7 +500,7 @@ private async ValueTask ProcessTranslationAsync((IEnumerable Texts, Fi this.logger.LogError(e, "翻訳中にエラーが発生"); var stopped = await this.dispatcher.InvokeAsync(() => { - if (hidden.Task.IsCompleted || this.disposedValue || !this.OverlayVisible) return false; + if (hidden.Task.IsCompleted || this.disposedValue) return false; this.captureEvents.Dispose(); ClearPendingCapture(); this.captureInput.Dispose(); @@ -526,7 +527,7 @@ private async ValueTask ProcessTranslationAsync((IEnumerable Texts, Fi cancellationToken.ThrowIfCancellationRequested(); await this.dispatcher.InvokeAsync(() => { - if (!hidden.Task.IsCompleted && !this.disposedValue && this.OverlayVisible) + if (!hidden.Task.IsCompleted && !this.disposedValue) { UpdateOcrTexts(texts, hidden.Task); } From 556c65bc3139fadb56d5fce411bc4bf08be2ae65 Mon Sep 17 00:00:00 2001 From: Freesia Date: Tue, 3 Nov 2026 23:34:55 +0900 Subject: [PATCH 6/8] =?UTF-8?q?=E9=9D=9E=E8=A1=A8=E7=A4=BA=E5=87=A6?= =?UTF-8?q?=E7=90=86=E3=81=AE=E4=B8=8D=E8=A6=81=E3=81=AA=E5=88=A4=E5=AE=9A?= =?UTF-8?q?=E3=82=92=E9=99=A4=E5=8E=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../OverlayVisibilityTests.cs | 120 ++++++++++++++++-- .../Modules/Main/MainViewModelBase.cs | 15 +-- 2 files changed, 113 insertions(+), 22 deletions(-) diff --git a/WindowTranslator.Tests/OverlayVisibilityTests.cs b/WindowTranslator.Tests/OverlayVisibilityTests.cs index 2bb9b11b..1505771a 100644 --- a/WindowTranslator.Tests/OverlayVisibilityTests.cs +++ b/WindowTranslator.Tests/OverlayVisibilityTests.cs @@ -83,9 +83,12 @@ private static async Task RunInStaAsync() await VerifyInFlightAsync(Stage.Ocr, withoutContext: true); await VerifyInFlightAsync(Stage.Ocr, failFirst: true); await VerifyInFlightAsync(Stage.Translation, failFirst: true); + await VerifyHiddenBeforeWorkStartsAsync(); await VerifyHiddenDuringResetAsync(); await VerifyHiddenDuringTrackingAsync(); + await VerifyHiddenDuringColorAsync(); await VerifyHiddenBeforeTranslationAsync(); + await VerifyOldEmptyResultDoesNotRemoveNewAsync(); await VerifyUiNotificationsAsync(); await VerifyReentrantShowAsync(); await VerifyReentrantHideAsync(); @@ -198,6 +201,55 @@ private static async Task VerifyHiddenDuringResetAsync() await WaitUntilAsync(() => fixture.Model.OcrTexts.Any(text => text.SourceText == "new")); } + private static async Task VerifyHiddenBeforeWorkStartsAsync() + { + await using var fixture = new Fixture(delayFilter: true); + const int blockedWorkers = 4; + using var started = new CountdownEvent(blockedWorkers); + using var release = new ManualResetEventSlim(); + ThreadPool.GetMinThreads(out var minWorker, out var minIo); + ThreadPool.GetMaxThreads(out var maxWorker, out var maxIo); + Task[] blockers = []; + try + { + Assert.True(ThreadPool.SetMaxThreads(Math.Max(maxWorker, blockedWorkers + 1), maxIo)); + Assert.True(ThreadPool.SetMinThreads(Math.Max(minWorker, blockedWorkers + 1), minIo)); + blockers = Enumerable.Range(0, blockedWorkers).Select(_ => Task.Run(() => + { + started.Signal(); + if (!release.Wait(TimeSpan.FromSeconds(15))) throw new TimeoutException(); + })).ToArray(); + Assert.True(started.Wait(TimeSpan.FromSeconds(5))); + Assert.True(ThreadPool.SetMinThreads(1, minIo)); + Assert.True(ThreadPool.SetMaxThreads(blockedWorkers, maxIo)); + fixture.SubmitCapture(Bitmap(320)); + fixture.Tracking.OnNext(([new("old", 0, 0, 100, 30, 20, false)], + new() { SoftwareBitmap = Bitmap(320), ImageSize = new(320, 240) })); + fixture.Translations.OnNext(([new("old", 0, 0, 100, 30, 20, false)], + new() { SoftwareBitmap = Bitmap(320), ImageSize = new(320, 240) })); + var timeout = Stopwatch.StartNew(); + while (fixture.HiddenSubscriberCount != 3 && timeout.Elapsed < TimeSpan.FromSeconds(5)) + { + await System.Windows.Threading.Dispatcher.Yield(System.Windows.Threading.DispatcherPriority.Background); + } + Assert.Equal(3, fixture.HiddenSubscriberCount); + fixture.Model.OverlayVisible = false; + fixture.Filter.Gate.Release.TrySetResult(); + } + finally + { + release.Set(); + ThreadPool.SetMinThreads(1, minIo); + ThreadPool.SetMaxThreads(maxWorker, maxIo); + ThreadPool.SetMinThreads(minWorker, minIo); + await Task.WhenAll(blockers).WaitAsync(TimeSpan.FromSeconds(5)); + } + await WaitUntilAsync(() => fixture.HiddenSubscriberCount == 0); + Assert.False(fixture.Ocr.Gate.Started.Task.IsCompleted); + Assert.False(fixture.Tracker.ResetStarted.Task.IsCompleted); + Assert.False(fixture.Filter.Gate.Started.Task.IsCompleted); + } + private static async Task VerifyHiddenDuringTrackingAsync() { await using var fixture = new Fixture(delayUpdate: true); @@ -213,6 +265,17 @@ private static async Task VerifyHiddenDuringTrackingAsync() await WaitUntilAsync(() => fixture.Model.OcrTexts.Any(text => text.SourceText == "new")); } + private static async Task VerifyHiddenDuringColorAsync() + { + await using var fixture = new Fixture(Stage.Color); + fixture.SubmitCapture(Bitmap(320)); + await fixture.Color.Gate.Started.Task.WaitAsync(TimeSpan.FromSeconds(5)); + fixture.Model.OverlayVisible = false; + fixture.Color.Gate.Release.TrySetResult(); + await WaitUntilAsync(() => fixture.Color.Bitmap is { } bitmap && IsDisposed(bitmap)); + Assert.Equal(0, fixture.Cache.ContainsCount); + } + private static async Task VerifyHiddenBeforeTranslationAsync() { await using var fixture = new Fixture(delayFilter: true); @@ -233,6 +296,27 @@ private static async Task VerifyHiddenBeforeTranslationAsync() await WaitUntilAsync(() => fixture.Model.OcrTexts.Any(text => text.SourceText == "new")); } + private static async Task VerifyOldEmptyResultDoesNotRemoveNewAsync() + { + await using var fixture = new Fixture(delayFilter: true, emptyFirstCapture: true, delayNextFilter: true); + fixture.SubmitCapture(Bitmap(320)); + await fixture.Filter.Gate.Started.Task.WaitAsync(TimeSpan.FromSeconds(5)); + fixture.Model.OverlayVisible = false; + fixture.Model.OverlayVisible = true; + fixture.SubmitCapture(Bitmap(321)); + await WaitUntilAsync(() => fixture.Model.OcrTexts.Any(text => text.SourceText == "new")); + fixture.Filter.Gate.Release.TrySetResult(); + await fixture.Filter.Gate.NextStarted.Task.WaitAsync(TimeSpan.FromSeconds(5)); + try + { + Assert.Contains(fixture.Model.OcrTexts, text => text.SourceText == "new"); + } + finally + { + fixture.Filter.Gate.NextRelease.TrySetResult(); + } + } + private static async Task VerifyUiNotificationsAsync() { await using var fixture = new Fixture(multipleTexts: true); @@ -327,7 +411,10 @@ private static async Task VerifyDelayedCaptureAsync() await using var fixture = new Fixture(); var handler = fixture.Capture.CurrentHandler!; fixture.Model.OverlayVisible = false; + Assert.Null(fixture.Capture.CurrentHandler); fixture.Model.OverlayVisible = true; + Assert.NotNull(fixture.Capture.CurrentHandler); + Assert.NotSame(handler, fixture.Capture.CurrentHandler); await handler.InvokeAsync(fixture.Capture, null!); await Task.Delay(150); Assert.False(fixture.Ocr.Gate.Started.Task.IsCompleted); @@ -454,13 +541,13 @@ private sealed class Fixture : System.IAsyncDisposable private readonly OcrTraceRecorder recorder = new(NullLogger.Instance); private readonly Action submitCapture; - public Fixture(Stage? delayed = null, double captureInterval = 0, bool delayReset = false, bool delayFilter = false, bool multipleTexts = false, bool withoutContext = false, bool failFirst = false, bool delayUpdate = false, bool isOneShotMode = false) + public Fixture(Stage? delayed = null, double captureInterval = 0, bool delayReset = false, bool delayFilter = false, bool multipleTexts = false, bool withoutContext = false, bool failFirst = false, bool delayUpdate = false, bool isOneShotMode = false, bool emptyFirstCapture = false, bool delayNextFilter = false) { - this.Ocr = new(delayed == Stage.Ocr, multipleTexts, failFirst && delayed == Stage.Ocr); + this.Ocr = new(delayed == Stage.Ocr, multipleTexts, failFirst && delayed == Stage.Ocr, emptyFirstCapture); this.Color = new(delayed == Stage.Color); this.Translator = new(delayed == Stage.Translation, failFirst && delayed == Stage.Translation); this.Tracker = new(delayReset, delayUpdate); - this.Filter = new(delayFilter); + this.Filter = new(delayFilter, delayNextFilter); var context = SynchronizationContext.Current; try { @@ -468,7 +555,7 @@ public Fixture(Stage? delayed = null, double captureInterval = 0, bool delayRese this.Model = new OverlayMainViewModel( null!, new Snapshot(new() { CaptureInterval = captureInterval, IsOneShotMode = isOneShotMode }), new ProcessInfo(), this.Capture, this.Ocr, new Snapshot(new()), this.Tracker, this.recorder, - this.Translator, new Cache(), this.Color, delayFilter ? [this.Filter] : [], NullLogger.Instance); + this.Translator, this.Cache, this.Color, delayFilter ? [this.Filter] : [], NullLogger.Instance); } finally { @@ -483,6 +570,7 @@ public Fixture(Stage? delayed = null, double captureInterval = 0, bool delayRese public Tracker Tracker { get; } public Color Color { get; } public Translator Translator { get; } + public Cache Cache { get; } = new(); public Filter Filter { get; } public OverlayMainViewModel Model { get; } public System.Windows.Threading.DispatcherTimer TrackingTimer => Field("trackingTimer"); @@ -490,6 +578,9 @@ public Fixture(Stage? delayed = null, double captureInterval = 0, bool delayRese => Field, FilterContext)>>("trackingRequests"); public Subject<(IEnumerable Texts, FilterContext Context)?> Translations => Field, FilterContext)?>>("translationRequests"); + public int HiddenSubscriberCount + => ((Delegate?)typeof(MainViewModelBase).GetField("OverlayHidden", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(this.Model))?.GetInvocationList().Length ?? 0; public void SubmitCapture(SoftwareBitmap bitmap) => this.submitCapture(bitmap); @@ -505,13 +596,14 @@ public async ValueTask DisposeAsync() } } - private sealed class Gate(bool delayed) + private sealed class Gate(bool delayed, bool delayNext = false) { private int count; public int Count => Volatile.Read(ref this.count); public TaskCompletionSource Started { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); public TaskCompletionSource NextStarted { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); public TaskCompletionSource Release { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource NextRelease { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); public async Task EnterAsync() { @@ -519,6 +611,7 @@ public async Task EnterAsync() if (current == 1) this.Started.TrySetResult(); else this.NextStarted.TrySetResult(); if (delayed && current == 1) await this.Release.Task.WaitAsync(TimeSpan.FromSeconds(10)).ConfigureAwait(false); + if (delayNext && current == 2) await this.NextRelease.Task.WaitAsync(TimeSpan.FromSeconds(10)).ConfigureAwait(false); } } @@ -544,13 +637,14 @@ private sealed class Capture : ICaptureModule public void StopCapture() => this.OnStopped?.Invoke(); } - private sealed class Ocr(bool delayed, bool multipleTexts, bool failFirst) : IOcrModule + private sealed class Ocr(bool delayed, bool multipleTexts, bool failFirst, bool emptyFirstCapture) : IOcrModule { public Gate Gate { get; } = new(delayed); public async ValueTask> RecognizeAsync(OcrCaptureInput input) { await this.Gate.EnterAsync(); if (failFirst && this.Gate.Count == 1) throw new InvalidOperationException("模擬 OCR エラー"); + if (emptyFirstCapture && input.Source.PixelWidth == 320) return []; var text = input.Source.PixelWidth == 321 ? "new" : "old"; var first = new TextRect(text, 0, 0, 100, 30, 20, false); return multipleTexts ? [first, first with { SourceText = "old2", Y = 40 }] : [first]; @@ -595,8 +689,10 @@ public void Reset() private sealed class Color(bool delayed) : IColorModule { public Gate Gate { get; } = new(delayed); + public SoftwareBitmap? Bitmap { get; private set; } public async ValueTask> ConvertColorAsync(SoftwareBitmap bitmap, IEnumerable texts) { + this.Bitmap = bitmap; var result = texts.ToArray(); await this.Gate.EnterAsync(); return result; @@ -614,9 +710,9 @@ public async ValueTask TranslateAsync(TextInfo[] srcTexts) } } - private sealed class Filter(bool delayed) : IFilterModule + private sealed class Filter(bool delayed, bool delayNext) : IFilterModule { - public Gate Gate { get; } = new(delayed); + public Gate Gate { get; } = new(delayed, delayNext); public async IAsyncEnumerable ExecutePreTranslate(IAsyncEnumerable texts, FilterContext context) { await this.Gate.EnterAsync(); @@ -628,7 +724,13 @@ public async IAsyncEnumerable ExecutePreTranslate(IAsyncEnumerable false; + private int containsCount; + public int ContainsCount => Volatile.Read(ref this.containsCount); + public bool Contains(string src) + { + Interlocked.Increment(ref this.containsCount); + return false; + } public string Get(string src) => throw new InvalidOperationException(); public void AddRange(IEnumerable<(string src, string dst)> pairs) { } } diff --git a/WindowTranslator/Modules/Main/MainViewModelBase.cs b/WindowTranslator/Modules/Main/MainViewModelBase.cs index 61180a6e..c58e8bdc 100644 --- a/WindowTranslator/Modules/Main/MainViewModelBase.cs +++ b/WindowTranslator/Modules/Main/MainViewModelBase.cs @@ -146,10 +146,9 @@ public MainViewModelBase( // タイマーの空通知では直前のOCR結果を再利用する。 .Scan(default((IEnumerable Texts, FilterContext Context)), (previous, current) => { - if (!this.OverlayVisible && current.Kind == NotificationKind.OnNext) + if (!this.OverlayVisible) { previous.Context?.SoftwareBitmap.Dispose(); - current.Value.Context?.SoftwareBitmap.Dispose(); return default; } if (current.Value.Context is { } context) @@ -172,10 +171,6 @@ public MainViewModelBase( { var hidden = new TaskCompletionSource(); using var visibility = ObserveHiding(hidden); - if (!this.OverlayVisible) - { - return; - } using var bitmap = request.Context.SoftwareBitmap.GetReadOnlyView(); var reset = this.needsTrackerReset; this.needsTrackerReset = false; @@ -316,7 +311,6 @@ private async ValueTask RecognizeAsync(SoftwareBitmap capturedBitmap, Cancellati using var bitmap = new DisposeAction(capturedBitmap.Dispose); var hidden = new TaskCompletionSource(); using var visibility = ObserveHiding(hidden); - if (!this.OverlayVisible) return; await Task.CompletedTask.ConfigureAwait(ConfigureAwaitOptions.ForceYielding); cancellationToken.ThrowIfCancellationRequested(); if (this.disposedValue || hidden.Task.IsCompleted) @@ -448,7 +442,6 @@ private async ValueTask ProcessTranslationAsync((IEnumerable Texts, Fi using var bitmap = request.Context.SoftwareBitmap; var hidden = new TaskCompletionSource(); using var visibility = ObserveHiding(hidden); - if (!this.OverlayVisible) return; await Task.CompletedTask.ConfigureAwait(ConfigureAwaitOptions.ForceYielding); cancellationToken.ThrowIfCancellationRequested(); if (this.disposedValue || hidden.Task.IsCompleted) @@ -466,10 +459,6 @@ private async ValueTask ProcessTranslationAsync((IEnumerable Texts, Fi using var t = this.logger.LogDebugTime("翻訳前処理"); texts = await tmp.ToArrayAsync(cancellationToken); } - if (hidden.Task.IsCompleted) - { - return; - } try { var requests = texts @@ -527,7 +516,7 @@ private async ValueTask ProcessTranslationAsync((IEnumerable Texts, Fi cancellationToken.ThrowIfCancellationRequested(); await this.dispatcher.InvokeAsync(() => { - if (!hidden.Task.IsCompleted && !this.disposedValue) + if (!this.disposedValue) { UpdateOcrTexts(texts, hidden.Task); } From 467a02bc45eea5907a23a8fa9d0d3ed289b249d7 Mon Sep 17 00:00:00 2001 From: Freesia Date: Wed, 4 Nov 2026 22:21:28 +0900 Subject: [PATCH 7/8] =?UTF-8?q?=E3=82=AA=E3=83=BC=E3=83=90=E3=83=BC?= =?UTF-8?q?=E3=83=AC=E3=82=A4=E9=9D=9E=E8=A1=A8=E7=A4=BA=E6=99=82=E3=81=AE?= =?UTF-8?q?=E8=BF=BD=E8=B7=A1=E3=82=92=E5=81=9C=E6=AD=A2=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../OverlayVisibilityTests.cs | 663 +++++++++++------- .../Modules/Main/MainViewModelBase.cs | 350 ++++----- .../Modules/Main/OverlayMainWindow.xaml | 6 +- .../Modules/Main/OverlayMainWindow.xaml.cs | 8 + 4 files changed, 542 insertions(+), 485 deletions(-) diff --git a/WindowTranslator.Tests/OverlayVisibilityTests.cs b/WindowTranslator.Tests/OverlayVisibilityTests.cs index 1505771a..590f3f4f 100644 --- a/WindowTranslator.Tests/OverlayVisibilityTests.cs +++ b/WindowTranslator.Tests/OverlayVisibilityTests.cs @@ -3,11 +3,14 @@ using System.Runtime.ExceptionServices; using System.Text; using System.Windows; +using System.Windows.Interop; +using System.Windows.Threading; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using Microsoft.VisualStudio.Threading; using R3; using Windows.Graphics.Imaging; +using WindowTranslator.Controls; using WindowTranslator.Modules; using WindowTranslator.Modules.Capture; using WindowTranslator.Modules.Main; @@ -69,8 +72,6 @@ private static async Task RunInStaAsync() { var application = new Application { ShutdownMode = ShutdownMode.OnExplicitShutdown }; Exception? failure = null; - Fixture? afterShutdown = null; - SoftwareBitmap? pendingAfterShutdown = null; _ = application.Dispatcher.InvokeAsync(async () => { try @@ -83,23 +84,24 @@ private static async Task RunInStaAsync() await VerifyInFlightAsync(Stage.Ocr, withoutContext: true); await VerifyInFlightAsync(Stage.Ocr, failFirst: true); await VerifyInFlightAsync(Stage.Translation, failFirst: true); - await VerifyHiddenBeforeWorkStartsAsync(); + await VerifyQueuedCapturesAsync(); + await VerifyOcrCompletesWhileHiddenAsync(); await VerifyHiddenDuringResetAsync(); await VerifyHiddenDuringTrackingAsync(); + await VerifyUiResponsiveDuringTrackingAsync(); + await VerifyUiResponsiveDuringTranslationErrorAsync(); await VerifyHiddenDuringColorAsync(); + await VerifyHiddenDuringCacheAsync(); + await VerifyOldHandoffAfterReopenAsync(); await VerifyHiddenBeforeTranslationAsync(); await VerifyOldEmptyResultDoesNotRemoveNewAsync(); - await VerifyUiNotificationsAsync(); - await VerifyReentrantShowAsync(); - await VerifyReentrantHideAsync(); - await VerifyNestedVisibilityAsync(); - await VerifyPendingAsync(); await VerifyOneShotAsync(); - await VerifyDelayedCaptureAsync(); + await VerifyAutomaticWindowVisibilityAsync(); + await VerifyOneShotWindowVisibilityAsync(); + await VerifyDelayedCaptureAsync(isOneShotMode: false); + await VerifyDelayedCaptureAsync(isOneShotMode: true); + await VerifyQueuedOneShotStopAsync(); await VerifyDisposeAsync(); - afterShutdown = new Fixture(captureInterval: 0.5); - pendingAfterShutdown = Bitmap(320); - afterShutdown.SubmitCapture(pendingAfterShutdown); } catch (Exception exception) { @@ -112,9 +114,6 @@ private static async Task RunInStaAsync() }); application.Run(); if (failure is not null) ExceptionDispatchInfo.Capture(failure).Throw(); - Task.Run(afterShutdown!.Model.Dispose).GetAwaiter().GetResult(); - Assert.True(IsDisposed(pendingAfterShutdown!)); - afterShutdown.DisposeAsync().AsTask().GetAwaiter().GetResult(); finished.TrySetResult(); } catch (Exception exception) @@ -131,30 +130,40 @@ private static async Task VerifyHiddenAsync() { await using var fixture = new Fixture(); SoftwareBitmap? tracked = null; + var hiddenTicks = 0; var hiddenTranslations = 0; using var tracking = fixture.Tracking.Subscribe(request => { - if (request.Context is { } context) tracked = context.SoftwareBitmap; + if (request.Context is { } context) Volatile.Write(ref tracked, context.SoftwareBitmap); + else if (!fixture.Model.OverlayVisible) + { + Interlocked.Increment(ref hiddenTicks); + } }); using var translations = fixture.Translations.Subscribe(request => { - if (request is not null && !fixture.Model.OverlayVisible) hiddenTranslations++; + if (request is not null && !fixture.Model.OverlayVisible) Interlocked.Increment(ref hiddenTranslations); }); fixture.SubmitCapture(Bitmap(320)); - await WaitUntilAsync(() => fixture.Translator.Gate.Count > 0 && tracked is not null); + await WaitUntilAsync(() => fixture.Translator.Gate.Count > 0 && Volatile.Read(ref tracked) is not null); fixture.Model.OverlayVisible = false; - Assert.False(fixture.TrackingTimer.IsEnabled); var trackerCount = fixture.Tracker.UpdateCount; var colorCount = fixture.Color.Gate.Count; - await Task.Delay(400); + await WaitUntilAsync(() => IsDisposed(Volatile.Read(ref tracked)!)); + await Task.Delay(150); + var ticksAfterStopping = Volatile.Read(ref hiddenTicks); + await Task.Delay(250); + Assert.Equal(ticksAfterStopping, Volatile.Read(ref hiddenTicks)); Assert.Equal(trackerCount, fixture.Tracker.UpdateCount); Assert.Equal(colorCount, fixture.Color.Gate.Count); - Assert.Equal(0, hiddenTranslations); - Assert.True(IsDisposed(tracked!)); + Assert.Equal(0, Volatile.Read(ref hiddenTranslations)); + Assert.True(IsDisposed(Volatile.Read(ref tracked)!)); fixture.Model.OverlayVisible = true; + await System.Windows.Threading.Dispatcher.Yield(System.Windows.Threading.DispatcherPriority.Background); Assert.Empty(fixture.Model.OcrTexts); - fixture.SubmitCapture(Bitmap(321)); + fixture.Ocr.Text = "new"; + fixture.SubmitCapture(Bitmap(320)); await WaitUntilAsync(() => fixture.Model.OcrTexts.Any(text => text.SourceText == "new")); Assert.DoesNotContain(fixture.Model.OcrTexts, text => text.SourceText == "old"); Assert.True(fixture.Tracker.ResetCount >= 2); @@ -201,53 +210,35 @@ private static async Task VerifyHiddenDuringResetAsync() await WaitUntilAsync(() => fixture.Model.OcrTexts.Any(text => text.SourceText == "new")); } - private static async Task VerifyHiddenBeforeWorkStartsAsync() + private static async Task VerifyQueuedCapturesAsync() { - await using var fixture = new Fixture(delayFilter: true); - const int blockedWorkers = 4; - using var started = new CountdownEvent(blockedWorkers); - using var release = new ManualResetEventSlim(); - ThreadPool.GetMinThreads(out var minWorker, out var minIo); - ThreadPool.GetMaxThreads(out var maxWorker, out var maxIo); - Task[] blockers = []; - try + await using (var throttled = new Fixture(captureInterval: 0.3)) { - Assert.True(ThreadPool.SetMaxThreads(Math.Max(maxWorker, blockedWorkers + 1), maxIo)); - Assert.True(ThreadPool.SetMinThreads(Math.Max(minWorker, blockedWorkers + 1), minIo)); - blockers = Enumerable.Range(0, blockedWorkers).Select(_ => Task.Run(() => - { - started.Signal(); - if (!release.Wait(TimeSpan.FromSeconds(15))) throw new TimeoutException(); - })).ToArray(); - Assert.True(started.Wait(TimeSpan.FromSeconds(5))); - Assert.True(ThreadPool.SetMinThreads(1, minIo)); - Assert.True(ThreadPool.SetMaxThreads(blockedWorkers, maxIo)); - fixture.SubmitCapture(Bitmap(320)); - fixture.Tracking.OnNext(([new("old", 0, 0, 100, 30, 20, false)], - new() { SoftwareBitmap = Bitmap(320), ImageSize = new(320, 240) })); - fixture.Translations.OnNext(([new("old", 0, 0, 100, 30, 20, false)], - new() { SoftwareBitmap = Bitmap(320), ImageSize = new(320, 240) })); - var timeout = Stopwatch.StartNew(); - while (fixture.HiddenSubscriberCount != 3 && timeout.Elapsed < TimeSpan.FromSeconds(5)) - { - await System.Windows.Threading.Dispatcher.Yield(System.Windows.Threading.DispatcherPriority.Background); - } - Assert.Equal(3, fixture.HiddenSubscriberCount); - fixture.Model.OverlayVisible = false; - fixture.Filter.Gate.Release.TrySetResult(); + var old = Bitmap(320); + throttled.SubmitCapture(old); + throttled.Model.OverlayVisible = false; + throttled.Model.OverlayVisible = true; + await WaitUntilAsync(() => IsDisposed(old)); + Assert.False(throttled.Ocr.Gate.Started.Task.IsCompleted); + throttled.SubmitCapture(Bitmap(321)); + await WaitUntilAsync(() => throttled.Model.OcrTexts.Any(text => text.SourceText == "new")); } - finally - { - release.Set(); - ThreadPool.SetMinThreads(1, minIo); - ThreadPool.SetMaxThreads(maxWorker, maxIo); - ThreadPool.SetMinThreads(minWorker, minIo); - await Task.WhenAll(blockers).WaitAsync(TimeSpan.FromSeconds(5)); + + await using (var queued = new Fixture(Stage.Ocr)) + { + queued.SubmitCapture(Bitmap(320)); + await queued.Ocr.Gate.Started.Task.WaitAsync(TimeSpan.FromSeconds(5)); + var old = Bitmap(322); + queued.SubmitCapture(old); + await Task.Delay(150); + queued.Model.OverlayVisible = false; + queued.Model.OverlayVisible = true; + queued.Ocr.Gate.Release.TrySetResult(); + await WaitUntilAsync(() => IsDisposed(old)); + Assert.False(queued.Ocr.Gate.NextStarted.Task.IsCompleted); + queued.SubmitCapture(Bitmap(321)); + await WaitUntilAsync(() => queued.Model.OcrTexts.Any(text => text.SourceText == "new")); } - await WaitUntilAsync(() => fixture.HiddenSubscriberCount == 0); - Assert.False(fixture.Ocr.Gate.Started.Task.IsCompleted); - Assert.False(fixture.Tracker.ResetStarted.Task.IsCompleted); - Assert.False(fixture.Filter.Gate.Started.Task.IsCompleted); } private static async Task VerifyHiddenDuringTrackingAsync() @@ -265,6 +256,55 @@ private static async Task VerifyHiddenDuringTrackingAsync() await WaitUntilAsync(() => fixture.Model.OcrTexts.Any(text => text.SourceText == "new")); } + private static async Task VerifyOcrCompletesWhileHiddenAsync() + { + await using var fixture = new Fixture(Stage.Ocr); + SoftwareBitmap? tracked = null; + using var tracking = fixture.Tracking.Subscribe(request => + { + if (request.Context is { } context) tracked = context.SoftwareBitmap; + }); + var image = Bitmap(320); + fixture.SubmitCapture(image); + await fixture.Ocr.Gate.Started.Task.WaitAsync(TimeSpan.FromSeconds(5)); + fixture.Model.OverlayVisible = false; + await Task.Delay(150); + fixture.Ocr.Gate.Release.TrySetResult(); + await WaitUntilAsync(() => IsDisposed(image)); + await Task.Delay(200); + Assert.True(tracked is null || IsDisposed(tracked)); + Assert.Equal(0, fixture.Tracker.UpdateCount); + Assert.Equal(0, fixture.Color.Gate.Count); + Assert.Equal(0, fixture.Translator.Gate.Count); + } + + private static async Task VerifyUiResponsiveDuringTrackingAsync() + { + await using var fixture = new Fixture(delayUpdate: true); + fixture.SubmitCapture(Bitmap(320)); + await fixture.Tracker.Updated.Task.WaitAsync(TimeSpan.FromSeconds(5)); + var dispatcher = Application.Current.Dispatcher; + var responsive = Task.Run(async () => + { + try + { + await Task.Delay(250); + var ping = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _ = dispatcher.InvokeAsync(() => ping.TrySetResult()); + return await Task.WhenAny(ping.Task, Task.Delay(1000)) == ping.Task; + } + finally + { + fixture.Tracker.UpdateRelease.Set(); + } + }); + fixture.Model.OverlayVisible = false; + fixture.Model.OverlayVisible = true; + fixture.SubmitCapture(Bitmap(321)); + Assert.True(await responsive.WaitAsync(TimeSpan.FromSeconds(5))); + await WaitUntilAsync(() => fixture.Model.OcrTexts.Any(text => text.SourceText == "new")); + } + private static async Task VerifyHiddenDuringColorAsync() { await using var fixture = new Fixture(Stage.Color); @@ -276,6 +316,76 @@ private static async Task VerifyHiddenDuringColorAsync() Assert.Equal(0, fixture.Cache.ContainsCount); } + private static async Task VerifyUiResponsiveDuringTranslationErrorAsync() + { + await using var fixture = new Fixture(Stage.Translation, failFirst: true); + fixture.SubmitCapture(Bitmap(320)); + await fixture.Translator.Gate.Started.Task.WaitAsync(TimeSpan.FromSeconds(5)); + fixture.Tracker.BlockNextUpdate(); + await fixture.Tracker.Blocked.Task.WaitAsync(TimeSpan.FromSeconds(5)); + var dispatcher = Application.Current.Dispatcher; + var responsive = Task.Run(async () => + { + try + { + await fixture.Translator.Failed.Task.WaitAsync(TimeSpan.FromSeconds(5)); + await Task.Delay(100); + var ping = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _ = dispatcher.InvokeAsync(() => ping.TrySetResult()); + return await Task.WhenAny(ping.Task, Task.Delay(1000)) == ping.Task; + } + finally + { + fixture.Tracker.UpdateRelease.Set(); + } + }); + fixture.Translator.Gate.Release.TrySetResult(); + Assert.True(await responsive.WaitAsync(TimeSpan.FromSeconds(5))); + Assert.False(await fixture.Translator.Failed.Task.WaitAsync(TimeSpan.FromSeconds(5))); + await fixture.Presentation.Shown.Task.WaitAsync(TimeSpan.FromSeconds(5)); + } + + private static async Task VerifyHiddenDuringCacheAsync() + { + await using var fixture = new Fixture(delayCache: true); + var hiddenTranslation = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + using var translations = fixture.Translations.Subscribe(request => + { + if (request is not null && !fixture.Model.OverlayVisible) hiddenTranslation.TrySetResult(); + }); + fixture.SubmitCapture(Bitmap(320)); + await WaitUntilAsync(() => fixture.Model.OcrTexts.Any(text => text.SourceText == "old")); + await fixture.Cache.Started.Task.WaitAsync(TimeSpan.FromSeconds(5)); + fixture.Model.OverlayVisible = false; + fixture.Cache.Release.Set(); + await Task.Delay(250); + Assert.False(hiddenTranslation.Task.IsCompleted); + + fixture.Model.OverlayVisible = true; + fixture.SubmitCapture(Bitmap(321)); + await WaitUntilAsync(() => fixture.Model.OcrTexts.Any(text => text.SourceText == "new")); + } + + private static async Task VerifyOldHandoffAfterReopenAsync() + { + await using var fixture = new Fixture(); + fixture.Ocr.Text = "handoff-old"; + fixture.SubmitCapture(Bitmap(320)); + await WaitUntilAsync(() => fixture.Capture.StopCount > 0 + || fixture.Model.OcrTexts.Any(text => text.SourceText == "handoff-old")); + var bitmap = fixture.Color.Bitmap!; + if (fixture.Capture.StopCount == 0) + { + fixture.Model.OverlayVisible = false; + fixture.Model.OverlayVisible = true; + } + await WaitUntilAsync(() => IsDisposed(bitmap)); + fixture.Ocr.Text = "new"; + fixture.SubmitCapture(Bitmap(321)); + await WaitUntilAsync(() => fixture.Model.OcrTexts.Any(text => text.SourceText == "new")); + Assert.DoesNotContain(fixture.Model.OcrTexts, text => text.SourceText == "handoff-old"); + } + private static async Task VerifyHiddenBeforeTranslationAsync() { await using var fixture = new Fixture(delayFilter: true); @@ -283,7 +393,7 @@ private static async Task VerifyHiddenBeforeTranslationAsync() await fixture.Filter.Gate.Started.Task.WaitAsync(TimeSpan.FromSeconds(5)); var pending = Bitmap(322); fixture.Translations.OnNext(([new("pending", 0, 0, 100, 30, 20, false)], - new() { SoftwareBitmap = pending, ImageSize = new(322, 240) })); + new() { SoftwareBitmap = pending, ImageSize = new(322, 240) }, fixture.Visibility)); fixture.Model.OverlayVisible = false; Assert.True(IsDisposed(pending)); fixture.Filter.Gate.Release.TrySetResult(); @@ -317,196 +427,206 @@ private static async Task VerifyOldEmptyResultDoesNotRemoveNewAsync() } } - private static async Task VerifyUiNotificationsAsync() - { - await using var fixture = new Fixture(multipleTexts: true); - var switched = false; - var addedAfterShow = new List(); - fixture.Model.OcrTexts.CollectionChanged += (_, change) => - { - if (change.NewItems is not { } items) return; - if (switched) - { - addedAfterShow.AddRange(items.Cast().Select(text => text.SourceText)); - } - else - { - switched = true; - fixture.Model.OverlayVisible = false; - fixture.Model.OverlayVisible = true; - } - }; - fixture.SubmitCapture(Bitmap(320)); - await WaitUntilAsync(() => switched); - await Task.Delay(100); - Assert.Empty(addedAfterShow); - Assert.Empty(fixture.Model.OcrTexts); - } - - private static async Task VerifyPendingAsync() - { - await using (var immediate = new Fixture()) - { - var image = Bitmap(322); - immediate.SubmitCapture(image); - immediate.Model.OverlayVisible = false; - await WaitUntilAsync(() => IsDisposed(image)); - } - - await using (var slow = new Fixture(captureInterval: (uint.MaxValue - 1) / 1000d)) - { - var pendingFrame = Bitmap(323); - slow.SubmitCapture(pendingFrame); - slow.Model.OverlayVisible = false; - Assert.True(IsDisposed(pendingFrame)); - } - - await using (var delayed = new Fixture(captureInterval: 0.1)) - { - delayed.SubmitCapture(Bitmap(320)); - Thread.Sleep(250); - delayed.Model.OverlayVisible = false; - delayed.Model.OverlayVisible = true; - delayed.SubmitCapture(Bitmap(321)); - await Task.Delay(30); - Assert.False(delayed.Ocr.Gate.Started.Task.IsCompleted); - await WaitUntilAsync(() => delayed.Model.OcrTexts.Any(text => text.SourceText == "new")); - } - - await using (var throttled = new Fixture(captureInterval: 0.5)) - { - var first = Bitmap(320); - var last = Bitmap(321); - throttled.SubmitCapture(first); - await Task.Delay(100); - throttled.SubmitCapture(last); - Assert.True(IsDisposed(first)); - await WaitUntilAsync(() => throttled.Model.OcrTexts.Any(text => text.SourceText == "new")); - Assert.DoesNotContain(throttled.Model.OcrTexts, text => text.SourceText == "old"); - var pendingFrame = Bitmap(323); - throttled.SubmitCapture(pendingFrame); - throttled.Model.OverlayVisible = false; - Assert.True(IsDisposed(pendingFrame)); - throttled.Model.OverlayVisible = true; - throttled.SubmitCapture(Bitmap(321)); - await WaitUntilAsync(() => throttled.Model.OcrTexts.Any(text => text.SourceText == "new")); - } - - await using var fixture = new Fixture(Stage.Ocr); - fixture.SubmitCapture(Bitmap(320)); - await fixture.Ocr.Gate.Started.Task.WaitAsync(TimeSpan.FromSeconds(5)); - var pending = Bitmap(322); - fixture.SubmitCapture(pending); - await Task.Delay(150); - fixture.Model.OverlayVisible = false; - Assert.True(IsDisposed(pending)); - fixture.Model.OverlayVisible = true; - fixture.Ocr.Gate.Release.TrySetResult(); - await Task.Delay(250); - Assert.False(fixture.Ocr.Gate.NextStarted.Task.IsCompleted); - } - - private static async Task VerifyDelayedCaptureAsync() + private static async Task VerifyDelayedCaptureAsync(bool isOneShotMode) { - await using var fixture = new Fixture(); + await using var fixture = new Fixture(isOneShotMode: isOneShotMode); var handler = fixture.Capture.CurrentHandler!; fixture.Model.OverlayVisible = false; Assert.Null(fixture.Capture.CurrentHandler); fixture.Model.OverlayVisible = true; Assert.NotNull(fixture.Capture.CurrentHandler); Assert.NotSame(handler, fixture.Capture.CurrentHandler); - await handler.InvokeAsync(fixture.Capture, null!); + var stops = fixture.Capture.StopCount; + var error = await Record.ExceptionAsync(() => handler.InvokeAsync(fixture.Capture, null!)); await Task.Delay(150); + Assert.Equal(stops, fixture.Capture.StopCount); + Assert.Null(error); Assert.False(fixture.Ocr.Gate.Started.Task.IsCompleted); } - private static async Task VerifyOneShotAsync() + private static async Task VerifyQueuedOneShotStopAsync() { - await using var fixture = new Fixture(captureInterval: 10, isOneShotMode: true); - fixture.SubmitCapture(Bitmap(321)); - await fixture.Ocr.Gate.Started.Task.WaitAsync(TimeSpan.FromSeconds(5)); - await WaitUntilAsync(() => fixture.Model.OcrTexts.Any(text => text.SourceText == "new")); - Assert.Equal(0, fixture.Tracker.UpdateCount); + await using var fixture = new Fixture(isOneShotMode: true); + var handler = fixture.Capture.CurrentHandler!; + using var submitted = new ManualResetEventSlim(); + var delayedCapture = Task.Run(() => + { + var operation = handler.InvokeAsync(fixture.Capture, null!); + submitted.Set(); + return operation; + }); + Assert.True(submitted.Wait(TimeSpan.FromSeconds(5))); + fixture.Model.OverlayVisible = false; + fixture.Model.OverlayVisible = true; + var stops = fixture.Capture.StopCount; + await delayedCapture.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(stops, fixture.Capture.StopCount); } - private static async Task VerifyReentrantShowAsync() + private static async Task VerifyOneShotAsync() { - await using var fixture = new Fixture(Stage.Ocr); - var oldShown = false; - fixture.Model.OcrTexts.CollectionChanged += (_, _) => - oldShown |= fixture.Model.OcrTexts.Any(text => text.SourceText == "old"); - fixture.SubmitCapture(Bitmap(320)); - await fixture.Ocr.Gate.Started.Task.WaitAsync(TimeSpan.FromSeconds(5)); - - fixture.Capture.OnStopped = () => - { - fixture.Capture.OnStopped = null; - fixture.Model.OverlayVisible = true; - }; + await using var fixture = new Fixture(Stage.Translation, captureInterval: 10, isOneShotMode: true); fixture.Model.OverlayVisible = false; + fixture.Model.OverlayVisible = true; fixture.SubmitCapture(Bitmap(321)); - fixture.Ocr.Gate.Release.TrySetResult(); - + await fixture.Translator.Gate.Started.Task.WaitAsync(TimeSpan.FromSeconds(5)); await WaitUntilAsync(() => fixture.Model.OcrTexts.Any(text => text.SourceText == "new")); - Assert.False(oldShown); - } - private static async Task VerifyDisposeAsync() - { - await using (var pending = new Fixture(captureInterval: 0.5)) + try { - var image = Bitmap(320); - pending.SubmitCapture(image); - await Task.Run(pending.Model.Dispose).WaitAsync(TimeSpan.FromSeconds(5)); - Assert.True(IsDisposed(image)); + fixture.Model.OverlayVisible = false; + fixture.Model.OverlayVisible = true; + fixture.SubmitCapture(Bitmap(320)); + await WaitUntilAsync(() => fixture.Model.OcrTexts.Any(text => text.SourceText == "old")); + Assert.DoesNotContain(fixture.Model.OcrTexts, text => text.SourceText == "new"); + Assert.Equal(0, fixture.Tracker.UpdateCount); } - - await using (var running = new Fixture(Stage.Ocr)) + finally { - var image = Bitmap(320); - running.SubmitCapture(image); - await running.Ocr.Gate.Started.Task.WaitAsync(TimeSpan.FromSeconds(5)); - await Task.Run(running.Model.Dispose).WaitAsync(TimeSpan.FromSeconds(5)); - running.Ocr.Gate.Release.TrySetResult(); - await WaitUntilAsync(() => IsDisposed(image)); + fixture.Translator.Gate.Release.TrySetResult(); } } - private static async Task VerifyReentrantHideAsync() + private static async Task VerifyAutomaticWindowVisibilityAsync() { await using var fixture = new Fixture(); - fixture.Model.OcrTexts.Add(new TextRect("old", 0, 0, 100, 30, 20, false)); fixture.Model.OverlayVisible = false; - var starts = fixture.Capture.StartCount; - fixture.Model.OcrTexts.CollectionChanged += (_, _) => fixture.Model.OverlayVisible = false; - - fixture.Model.OverlayVisible = true; + var target = new Window(); + var handle = new WindowInteropHelper(target).EnsureHandle(); + var desktop = new TestDesktopManager(); + var window = CreateOverlayWindow(null, handle, desktop, isOneShotMode: false); + try + { + Assert.False(fixture.Model.OverlayVisible); + Assert.Null(fixture.Capture.CurrentHandler); + window.Show(); + StopPositionTimer(window); + var overlay = (OverlayTextsControl)window.FindName("overlay"); + Assert.False(fixture.Model.OverlayVisible); + window.DataContext = fixture.Model; + Assert.True(fixture.Model.OverlayVisible); - Assert.False(fixture.Model.OverlayVisible); - Assert.Equal(starts, fixture.Capture.StartCount); + fixture.SubmitCapture(Bitmap(320)); + await WaitUntilAsync(() => fixture.Model.OcrTexts.Any(text => text.SourceText == "old") + && fixture.Translator.Gate.Count > 0); + + desktop.IsCurrent = false; + typeof(OverlayMainWindow).GetMethod("UpdateWindowPositionAndSize", BindingFlags.Instance | BindingFlags.NonPublic)! + .Invoke(window, null); + Assert.False(window.IsVisible); + Assert.Equal(Visibility.Visible, overlay.Visibility); + Assert.False(fixture.Model.OverlayVisible); + Assert.Null(fixture.Capture.CurrentHandler); + + overlay.SetCurrentValue(UIElement.VisibilityProperty, Visibility.Hidden); + overlay.SetCurrentValue(UIElement.VisibilityProperty, Visibility.Visible); + Assert.False(fixture.Model.OverlayVisible); + Assert.Null(fixture.Capture.CurrentHandler); + var trackerCount = fixture.Tracker.UpdateCount; + var colorCount = fixture.Color.Gate.Count; + var translationCount = fixture.Translator.Gate.Count; + await Task.Delay(350); + Assert.Equal(trackerCount, fixture.Tracker.UpdateCount); + Assert.Equal(colorCount, fixture.Color.Gate.Count); + Assert.Equal(translationCount, fixture.Translator.Gate.Count); + + window.SetCurrentValue(UIElement.VisibilityProperty, Visibility.Visible); + Assert.True(fixture.Model.OverlayVisible); + await Dispatcher.Yield(DispatcherPriority.Background); + Assert.Empty(fixture.Model.OcrTexts); + fixture.Ocr.Text = "new"; + fixture.SubmitCapture(Bitmap(321)); + await WaitUntilAsync(() => fixture.Model.OcrTexts.Any(text => text.SourceText == "new")); + Assert.DoesNotContain(fixture.Model.OcrTexts, text => text.SourceText == "old"); + + desktop.IsCurrent = true; + typeof(OverlayMainWindow).GetMethod("UpdateWindowPositionAndSize", BindingFlags.Instance | BindingFlags.NonPublic)! + .Invoke(window, null); + Assert.False(window.IsVisible); + Assert.False(fixture.Model.OverlayVisible); + window.SetCurrentValue(UIElement.VisibilityProperty, Visibility.Visible); + + overlay.SetCurrentValue(UIElement.VisibilityProperty, Visibility.Hidden); + Assert.False(fixture.Model.OverlayVisible); + window.SetCurrentValue(UIElement.VisibilityProperty, Visibility.Hidden); + window.SetCurrentValue(UIElement.VisibilityProperty, Visibility.Visible); + Assert.False(fixture.Model.OverlayVisible); + overlay.SetCurrentValue(UIElement.VisibilityProperty, Visibility.Visible); + Assert.True(fixture.Model.OverlayVisible); + + var stopsAtDispose = -1; + window.Closed += (_, _) => + { + fixture.Model.Dispose(); + stopsAtDispose = fixture.Capture.StopCount; + }; + window.Close(); + Assert.Equal(stopsAtDispose, fixture.Capture.StopCount); + } + finally + { + window.Close(); + target.Close(); + } } - private static async Task VerifyNestedVisibilityAsync() + private static async Task VerifyOneShotWindowVisibilityAsync() { - await using var fixture = new Fixture(); - fixture.Model.OcrTexts.Add(new TextRect("old", 0, 0, 100, 30, 20, false)); - fixture.Model.OverlayVisible = false; - var starts = fixture.Capture.StartCount; - var toggled = false; - fixture.Model.OcrTexts.CollectionChanged += (_, _) => + await using var fixture = new Fixture(isOneShotMode: true); + var target = new Window(); + var handle = new WindowInteropHelper(target).EnsureHandle(); + var window = CreateOverlayWindow(fixture.Model, handle, new TestDesktopManager(), isOneShotMode: true); + try { - if (toggled) return; - toggled = true; - fixture.Model.OverlayVisible = false; - fixture.Model.OverlayVisible = true; + Assert.False(fixture.Model.OverlayVisible); + window.Show(); + StopPositionTimer(window); + var overlay = (OverlayTextsControl)window.FindName("overlay"); + Assert.False(fixture.Model.OverlayVisible); + Assert.Null(fixture.Capture.CurrentHandler); + + overlay.SetCurrentValue(UIElement.VisibilityProperty, Visibility.Visible); + Assert.True(fixture.Model.OverlayVisible); + Assert.NotNull(fixture.Capture.CurrentHandler); + fixture.SubmitCapture(Bitmap(321)); + await WaitUntilAsync(() => fixture.Model.OcrTexts.Any(text => text.SourceText == "new")); + overlay.SetCurrentValue(UIElement.VisibilityProperty, Visibility.Hidden); + Assert.False(fixture.Model.OverlayVisible); + } + finally + { + window.Close(); + target.Close(); + } + } + + private static OverlayMainWindow CreateOverlayWindow( + OverlayMainViewModel? model, IntPtr target, IVirtualDesktopManager desktop, bool isOneShotMode) + => new(new Snapshot(new()), + new Snapshot(new() { IsOneShotMode = isOneShotMode }), + new ProcessInfo(target), desktop, NullLogger.Instance) + { + ShowActivated = false, + Left = -10000, + Top = -10000, + Width = 1, + Height = 1, + DataContext = model, }; - fixture.Model.OcrTexts.CollectionChanged += (_, _) => { }; - fixture.Model.OverlayVisible = true; - await Task.Delay(50); + private static void StopPositionTimer(OverlayMainWindow window) + => ((DispatcherTimer)typeof(OverlayMainWindow).GetField("timer", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(window)!).Stop(); - Assert.True(fixture.Model.OverlayVisible); - Assert.Equal(starts + 1, fixture.Capture.StartCount); + private static async Task VerifyDisposeAsync() + { + await using var fixture = new Fixture(Stage.Ocr); + var image = Bitmap(320); + fixture.SubmitCapture(image); + await fixture.Ocr.Gate.Started.Task.WaitAsync(TimeSpan.FromSeconds(5)); + await Task.Run(fixture.Model.Dispose).WaitAsync(TimeSpan.FromSeconds(5)); + fixture.Ocr.Gate.Release.TrySetResult(); + await WaitUntilAsync(() => IsDisposed(image)); } private static SoftwareBitmap Bitmap(int width) => new(BitmapPixelFormat.Bgra8, width, 240); @@ -539,21 +659,23 @@ private enum Stage { Ocr, Color, Translation } private sealed class Fixture : System.IAsyncDisposable { private readonly OcrTraceRecorder recorder = new(NullLogger.Instance); - private readonly Action submitCapture; - public Fixture(Stage? delayed = null, double captureInterval = 0, bool delayReset = false, bool delayFilter = false, bool multipleTexts = false, bool withoutContext = false, bool failFirst = false, bool delayUpdate = false, bool isOneShotMode = false, bool emptyFirstCapture = false, bool delayNextFilter = false) + public Fixture(Stage? delayed = null, double captureInterval = 0, bool delayReset = false, bool delayFilter = false, bool withoutContext = false, bool failFirst = false, bool delayUpdate = false, bool isOneShotMode = false, bool emptyFirstCapture = false, bool delayNextFilter = false, bool delayCache = false) { - this.Ocr = new(delayed == Stage.Ocr, multipleTexts, failFirst && delayed == Stage.Ocr, emptyFirstCapture); + this.Ocr = new(delayed == Stage.Ocr, failFirst && delayed == Stage.Ocr, emptyFirstCapture); this.Color = new(delayed == Stage.Color); this.Translator = new(delayed == Stage.Translation, failFirst && delayed == Stage.Translation); this.Tracker = new(delayReset, delayUpdate); this.Filter = new(delayFilter, delayNextFilter); + this.Cache = new(delayCache); + var presentation = DispatchProxy.Create(); + this.Presentation = (PresentationProxy)presentation; var context = SynchronizationContext.Current; try { if (withoutContext) SynchronizationContext.SetSynchronizationContext(null); this.Model = new OverlayMainViewModel( - null!, new Snapshot(new() { CaptureInterval = captureInterval, IsOneShotMode = isOneShotMode }), new ProcessInfo(), + presentation, new Snapshot(new() { CaptureInterval = captureInterval, IsOneShotMode = isOneShotMode }), new ProcessInfo(), this.Capture, this.Ocr, new Snapshot(new()), this.Tracker, this.recorder, this.Translator, this.Cache, this.Color, delayFilter ? [this.Filter] : [], NullLogger.Instance); } @@ -561,8 +683,6 @@ public Fixture(Stage? delayed = null, double captureInterval = 0, bool delayRese { if (withoutContext) SynchronizationContext.SetSynchronizationContext(context); } - this.submitCapture = typeof(MainViewModelBase).GetMethod("QueueCapture", BindingFlags.Instance | BindingFlags.NonPublic)! - .CreateDelegate>(this.Model); } public Capture Capture { get; } = new(); @@ -570,19 +690,18 @@ public Fixture(Stage? delayed = null, double captureInterval = 0, bool delayRese public Tracker Tracker { get; } public Color Color { get; } public Translator Translator { get; } - public Cache Cache { get; } = new(); + public Cache Cache { get; } public Filter Filter { get; } + public PresentationProxy Presentation { get; } public OverlayMainViewModel Model { get; } - public System.Windows.Threading.DispatcherTimer TrackingTimer => Field("trackingTimer"); - public Subject<(IEnumerable Texts, FilterContext Context)> Tracking - => Field, FilterContext)>>("trackingRequests"); - public Subject<(IEnumerable Texts, FilterContext Context)?> Translations - => Field, FilterContext)?>>("translationRequests"); - public int HiddenSubscriberCount - => ((Delegate?)typeof(MainViewModelBase).GetField("OverlayHidden", BindingFlags.Instance | BindingFlags.NonPublic)! - .GetValue(this.Model))?.GetInvocationList().Length ?? 0; + public Subject<(IEnumerable? Texts, FilterContext? Context, Observer? Visibility)> Tracking + => Field?, FilterContext?, Observer?)>>("trackingRequests"); + public Subject<(IEnumerable Texts, FilterContext Context, Observer Visibility)?> Translations + => Field, FilterContext, Observer)?>>("translationRequests"); + public Observer Visibility => (Observer)Field("captureLoop").Disposable!; - public void SubmitCapture(SoftwareBitmap bitmap) => this.submitCapture(bitmap); + public void SubmitCapture(SoftwareBitmap bitmap) + => Field)>>("captureRequests").OnNext((bitmap, this.Visibility)); private T Field(string name) where T : class => (T)typeof(MainViewModelBase).GetField(name, BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(this.Model)!; @@ -593,6 +712,7 @@ public async ValueTask DisposeAsync() await this.recorder.DisposeAsync(); this.Tracker.ResetRelease.Dispose(); this.Tracker.UpdateRelease.Dispose(); + this.Cache.Release.Dispose(); } } @@ -621,33 +741,41 @@ private sealed class Snapshot(T value) : IOptionsSnapshot where T : class public T Get(string? name) => this.Value; } - private sealed class ProcessInfo : IProcessInfoStore + private sealed class ProcessInfo(IntPtr handle = default) : IProcessInfoStore { - public IntPtr MainWindowHandle => IntPtr.Zero; + public IntPtr MainWindowHandle => handle; public string Name => "test"; } + private sealed class TestDesktopManager : IVirtualDesktopManager + { + public bool IsCurrent { get; set; } = true; + public bool IsWindowOnCurrentVirtualDesktop(IntPtr topLevelWindow) => this.IsCurrent; + public Guid GetWindowDesktopId(IntPtr hWnd) => Guid.Empty; + public int MoveWindowToDesktop(IntPtr hWnd, ref Guid desktop) => 0; + } + private sealed class Capture : ICaptureModule { + private int stops; public event AsyncEventHandler? Captured; public AsyncEventHandler? CurrentHandler => this.Captured; - public Action? OnStopped { get; set; } - public int StartCount { get; private set; } - public void StartCapture(IntPtr targetWindow) => this.StartCount++; - public void StopCapture() => this.OnStopped?.Invoke(); + public int StopCount => Volatile.Read(ref this.stops); + public void StartCapture(IntPtr targetWindow) { } + public void StopCapture() => Interlocked.Increment(ref this.stops); } - private sealed class Ocr(bool delayed, bool multipleTexts, bool failFirst, bool emptyFirstCapture) : IOcrModule + private sealed class Ocr(bool delayed, bool failFirst, bool emptyFirstCapture) : IOcrModule { public Gate Gate { get; } = new(delayed); + public string? Text { get; set; } public async ValueTask> RecognizeAsync(OcrCaptureInput input) { await this.Gate.EnterAsync(); if (failFirst && this.Gate.Count == 1) throw new InvalidOperationException("模擬 OCR エラー"); if (emptyFirstCapture && input.Source.PixelWidth == 320) return []; - var text = input.Source.PixelWidth == 321 ? "new" : "old"; - var first = new TextRect(text, 0, 0, 100, 30, 20, false); - return multipleTexts ? [first, first with { SourceText = "old2", Y = 40 }] : [first]; + var text = this.Text ?? (input.Source.PixelWidth == 321 ? "new" : "old"); + return [new TextRect(text, 0, 0, 100, 30, 20, false)]; } } @@ -655,20 +783,24 @@ private sealed class Tracker(bool delayReset, bool delayUpdate) : IOcrTextTracke { private int updates; private int resets; + private int blockNext; private TextRect[] previous = []; public TaskCompletionSource ResetStarted { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); public ManualResetEventSlim ResetRelease { get; } = new(); public ManualResetEventSlim UpdateRelease { get; } = new(); public TaskCompletionSource Updated { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource Blocked { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); public int UpdateCount => Volatile.Read(ref this.updates); public int ResetCount => Volatile.Read(ref this.resets); + public void BlockNextUpdate() => Interlocked.Exchange(ref this.blockNext, 1); public IReadOnlyList Update(IEnumerable observations, System.Drawing.Size imageSize, TimeSpan? timestamp = null) { var count = Interlocked.Increment(ref this.updates); this.Updated.TrySetResult(); - if (delayUpdate && count == 1 && !this.UpdateRelease.Wait(TimeSpan.FromSeconds(10))) + if ((delayUpdate && count == 1) || Interlocked.Exchange(ref this.blockNext, 0) == 1) { - throw new TimeoutException(); + this.Blocked.TrySetResult(); + if (!this.UpdateRelease.Wait(TimeSpan.FromSeconds(10))) throw new TimeoutException(); } var current = observations.ToArray(); this.previous = [.. this.previous.Where(old => current.All(text => text.SourceText != old.SourceText)), .. current]; @@ -702,14 +834,34 @@ public async ValueTask> ConvertColorAsync(SoftwareBitmap b private sealed class Translator(bool delayed, bool failFirst) : ITranslateModule { public Gate Gate { get; } = new(delayed); + public TaskCompletionSource Failed { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); public async ValueTask TranslateAsync(TextInfo[] srcTexts) { await this.Gate.EnterAsync(); - if (failFirst && this.Gate.Count == 1) throw new InvalidOperationException("模擬翻訳エラー"); + if (failFirst && this.Gate.Count == 1) + { + this.Failed.TrySetResult(Application.Current.Dispatcher.CheckAccess()); + throw new InvalidOperationException("模擬翻訳エラー"); + } return srcTexts.Select(text => text.SourceText).ToArray(); } } + public class PresentationProxy : DispatchProxy + { + public TaskCompletionSource Shown { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + protected override object? Invoke(MethodInfo? method, object?[]? args) + { + if (method?.Name == nameof(IPresentationService.OpenErrorReportDialogAsync)) + { + this.Shown.TrySetResult(); + return Task.FromResult(true); + } + throw new NotSupportedException(method?.Name); + } + } + private sealed class Filter(bool delayed, bool delayNext) : IFilterModule { public Gate Gate { get; } = new(delayed, delayNext); @@ -722,13 +874,20 @@ public async IAsyncEnumerable ExecutePreTranslate(IAsyncEnumerable ExecutePostTranslate(IAsyncEnumerable texts, FilterContext context) => texts; } - private sealed class Cache : ICacheModule + private sealed class Cache(bool delayed) : ICacheModule { private int containsCount; public int ContainsCount => Volatile.Read(ref this.containsCount); + public TaskCompletionSource Started { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public ManualResetEventSlim Release { get; } = new(); public bool Contains(string src) { - Interlocked.Increment(ref this.containsCount); + var count = Interlocked.Increment(ref this.containsCount); + if (delayed && count == 2) + { + this.Started.TrySetResult(); + if (!this.Release.Wait(TimeSpan.FromSeconds(10))) throw new TimeoutException(); + } return false; } public string Get(string src) => throw new InvalidOperationException(); diff --git a/WindowTranslator/Modules/Main/MainViewModelBase.cs b/WindowTranslator/Modules/Main/MainViewModelBase.cs index c58e8bdc..c4e60135 100644 --- a/WindowTranslator/Modules/Main/MainViewModelBase.cs +++ b/WindowTranslator/Modules/Main/MainViewModelBase.cs @@ -23,17 +23,14 @@ namespace WindowTranslator.Modules.Main; [ObservableObject] public abstract partial class MainViewModelBase : IDisposable { - private readonly Dispatcher dispatcher = Application.Current.Dispatcher; - private readonly Subject captureRequests = new(); - private readonly Subject recognitionRequests = new(); - private readonly Subject<(IEnumerable Texts, FilterContext Context)> trackingRequests = new(); - private readonly Subject<(IEnumerable Texts, FilterContext Context)?> translationRequests = new(); - private IDisposable captureEvents; - private readonly IDisposable captureInput; - private readonly IDisposable captureLoop; + private readonly Subject<(SoftwareBitmap Bitmap, Observer Visibility)> captureRequests = new(); + private readonly Subject<(IEnumerable? Texts, FilterContext? Context, Observer? Visibility)> trackingRequests = new(); + private readonly Subject<(IEnumerable Texts, FilterContext Context, Observer Visibility)?> translationRequests = new(); + private readonly Subject<(IEnumerable Texts, Observer? Visibility)> uiRequests = new(); + private readonly SerialDisposable captureLoop = new(); private readonly IDisposable trackingLoop; private readonly IDisposable translationLoop; - private readonly DispatcherTimer trackingTimer; + private readonly IDisposable uiLoop; private readonly IOcrModule ocr; private readonly List priorityRects; private readonly IOcrTextTracker ocrTextTracker; @@ -51,7 +48,6 @@ public abstract partial class MainViewModelBase : IDisposable private readonly double overlayOpacity; private readonly double mousePointerHitTestPadding; private readonly bool isOneShotMode; - private readonly TimeSpan captureInterval; [ObservableProperty] private string title; @@ -69,12 +65,8 @@ public abstract partial class MainViewModelBase : IDisposable public BusyScope Recognizing { get; } = new(); public BusyScope Filtering { get; } = new(); - private bool isFirstCapture = true; - private bool needsTrackerReset; - private bool updatingOcrTexts; - private SoftwareBitmap? pendingCapture; + private volatile bool isFirstCapture = true; private volatile bool disposedValue; - private event Action? OverlayHidden; public ObservableCollection OcrTexts { get; } = []; public string Font { get; } @@ -103,10 +95,8 @@ public MainViewModelBase( this.overlayOpacity = options.Value.OverlayOpacity; this.mousePointerHitTestPadding = options.Value.MousePointerHitTestPadding; this.isOneShotMode = options.Value.IsOneShotMode; - this.captureInterval = TimeSpan.FromSeconds(options.Value.CaptureInterval); this.DisplayBusy = options.Value.DisplayBusy; this.capture = capture ?? throw new ArgumentNullException(nameof(capture)); - this.captureEvents = CreateCaptureEvents(); this.ocr = ocr ?? throw new ArgumentNullException(nameof(ocr)); this.priorityRects = ocrParam.Value.PriorityRects ?? []; this.ocrTextTracker = ocrTextTracker ?? throw new ArgumentNullException(nameof(ocrTextTracker)); @@ -116,50 +106,50 @@ public MainViewModelBase( this.color = color ?? throw new ArgumentNullException(nameof(color)); this.filters = filters.ToArray(); this.logger = logger; - // 処理開始と非表示通知を UI スレッドで順序付ける。 - this.captureLoop = this.dispatcher.Invoke(() => this.recognitionRequests - .SubscribeAwaitThrottleFirstLast( - (bitmap, ct) => bitmap is null ? ValueTask.CompletedTask : RecognizeAsync(bitmap, ct), - static bitmap => bitmap?.Dispose(), configureAwait: true)); - this.captureInput = !this.isOneShotMode && this.captureInterval > TimeSpan.Zero - ? this.captureRequests.ThrottleLast(this.captureInterval).Subscribe(bitmap => - { - _ = this.dispatcher.InvokeAsync(() => - { - if (ReferenceEquals(Interlocked.CompareExchange(ref this.pendingCapture, null, bitmap), bitmap)) - { - this.recognitionRequests.OnNext(bitmap); - } - }); - }) - : Disposable.Empty; - this.trackingTimer = new DispatcherTimer(DispatcherPriority.Normal, this.dispatcher) - { - Interval = TimeSpan.FromMilliseconds(100), - }; - this.trackingTimer.Tick += (_, _) => this.trackingRequests.OnNext(default); + this.uiLoop = this.uiRequests + .ObserveOn(new DispatcherSynchronizationContext(Application.Current.Dispatcher)) + .Where(request => request.Visibility?.IsDisposed != true) + .Subscribe(request => UpdateOcrTexts(request.Texts)); + var captureInterval = TimeSpan.FromSeconds(options.Value.CaptureInterval); + var captures = !this.isOneShotMode && captureInterval > TimeSpan.Zero + ? this.captureRequests.ThrottleLast(captureInterval) + : this.captureRequests; this.trackingLoop = Disposable.Combine( Disposable.Create(() => this.trackingRequests.OnCompleted()), + captures.ObserveOnThreadPool() + .SubscribeAwait(RecognizeAsync, AwaitOperation.ThrottleFirstLast, configureAwait: false), this.trackingRequests .Materialize() .Synchronize() // タイマーの空通知では直前のOCR結果を再利用する。 - .Scan(default((IEnumerable Texts, FilterContext Context)), (previous, current) => + .Scan(default((IEnumerable? Texts, FilterContext? Context, Observer? Visibility)), (previous, current) => { - if (!this.OverlayVisible) + if (current.Kind != NotificationKind.OnNext || previous.Visibility?.IsDisposed == true) { previous.Context?.SoftwareBitmap.Dispose(); + previous = default; + } + if (current.Kind != NotificationKind.OnNext) + { return default; } if (current.Value.Context is { } context) { - if (previous.Context?.ImageSize != context.ImageSize) + if (current.Value.Visibility!.IsDisposed) { - this.needsTrackerReset = true; + context.SoftwareBitmap.Dispose(); + return previous; + } + if (previous.Context is { } previousContext && previousContext.ImageSize != context.ImageSize) + { + this.ocrTextTracker.Reset(); + } + if (this.isOneShotMode) + { + this.isFirstCapture = true; } - this.isFirstCapture |= this.isOneShotMode; } - else if (current.Kind == NotificationKind.OnNext) + else { return previous; } @@ -169,39 +159,31 @@ public MainViewModelBase( .Where(request => request.Context is not null) .SubscribeAwait(async (request, ct) => { - var hidden = new TaskCompletionSource(); - using var visibility = ObserveHiding(hidden); - using var bitmap = request.Context.SoftwareBitmap.GetReadOnlyView(); - var reset = this.needsTrackerReset; - this.needsTrackerReset = false; - // 重い追跡処理は UI スレッドを離れて実行する。 - await Task.CompletedTask.ConfigureAwait(ConfigureAwaitOptions.ForceYielding); + using var bitmap = request.Context!.SoftwareBitmap.GetReadOnlyView(); ct.ThrowIfCancellationRequested(); - if (hidden.Task.IsCompleted) return; - if (reset) this.ocrTextTracker.Reset(); - if (hidden.Task.IsCompleted) return; - var texts = await ProcessTrackingAsync(request, bitmap, hidden.Task, ct); - if (texts is null) - { - return; - } - await this.dispatcher.InvokeAsync(() => + if (this.isFirstCapture) this.ocrTextTracker.Reset(); + if (request.Visibility!.IsDisposed) return; + var texts = await ProcessTrackingAsync((request.Texts!, request.Context), bitmap, request.Visibility, ct); + if (texts is null) return; + if (request.Visibility.IsDisposed || this.disposedValue) return; + if (this.isFirstCapture) { - if (hidden.Task.IsCompleted || this.disposedValue) return; - if (this.isFirstCapture) + this.isFirstCapture = false; + if (request.Visibility.IsDisposed) { - this.isFirstCapture = false; - UpdateOcrTexts(texts, hidden.Task); + // 非表示前の処理で再表示後の初回フラグを消さない。 + this.isFirstCapture = true; + return; } - if (hidden.Task.IsCompleted) return; - this.translationRequests.OnNext((texts, request.Context with { SoftwareBitmap = bitmap.GetReadOnlyView(), })); - }); + this.uiRequests.OnNext((texts, request.Visibility)); + } + this.translationRequests.OnNext((texts, request.Context with { SoftwareBitmap = bitmap.GetReadOnlyView(), }, request.Visibility)); }, AwaitOperation.Drop, configureAwait: false, cancelOnCompleted: true)); - this.translationLoop = this.dispatcher.Invoke(() => this.translationRequests + this.translationLoop = this.translationRequests .SubscribeAwaitThrottleFirstLast( (request, ct) => request is { } value ? ProcessTranslationAsync(value, ct) : ValueTask.CompletedTask, - static request => request?.Context.SoftwareBitmap.Dispose(), configureAwait: true)); - this.trackingTimer.Start(); + static request => request?.Context.SoftwareBitmap.Dispose(), configureAwait: false); + this.captureLoop.Disposable = CreateCaptureEvents(); if (!this.isOneShotMode) { this.capture.StartCapture(this.processInfoStore.MainWindowHandle); @@ -210,113 +192,65 @@ await this.dispatcher.InvokeAsync(() => this.title = $"{this.name} - {this.translator.Name} ({transAsm.GetName().Version})"; } - private IDisposable ObserveHiding(TaskCompletionSource hidden) - { - void Handler() => hidden.TrySetResult(); - this.OverlayHidden += Handler; - return Disposable.Create(() => this.OverlayHidden -= Handler); - } - - private void QueueCapture(SoftwareBitmap bitmap) - { - if (this.isOneShotMode || this.captureInterval == TimeSpan.Zero) - { - this.recognitionRequests.OnNext(bitmap); - return; - } - Interlocked.Exchange(ref this.pendingCapture, bitmap)?.Dispose(); - this.captureRequests.OnNext(bitmap); - } - - private void ClearPendingCapture() - => Interlocked.Exchange(ref this.pendingCapture, null)?.Dispose(); - partial void OnOverlayVisibleChanged(bool value) { if (value) { + this.uiRequests.OnNext(([], null)); this.isFirstCapture = !this.isOneShotMode; - this.captureEvents = CreateCaptureEvents(); - if (this.updatingOcrTexts) - { - _ = this.dispatcher.InvokeAsync(this.OcrTexts.Clear, DispatcherPriority.Send); - } - else if (this.OcrTexts.Count > 0) - { - this.OcrTexts.Clear(); - } - if (this.OverlayVisible && !this.trackingTimer.IsEnabled) - { - this.trackingTimer.Start(); - // Start capture when overlay becomes visible - this.capture.StartCapture(this.processInfoStore.MainWindowHandle); - } + this.captureLoop.Disposable = CreateCaptureEvents(); + this.capture.StartCapture(this.processInfoStore.MainWindowHandle); } else { - this.OverlayHidden?.Invoke(); - this.captureEvents.Dispose(); - ClearPendingCapture(); - this.recognitionRequests.OnNext(null); + this.captureLoop.Disposable = null; this.translationRequests.OnNext(null); - this.trackingTimer.Stop(); - this.trackingRequests.OnNext(default); - // Stop capture when overlay becomes hidden + ThreadPool.QueueUserWorkItem(static model => model.trackingRequests.OnNext(default), this, preferLocal: false); this.capture.StopCapture(); this.logger.LogDebug("Overlay became hidden - capture stopped"); } } private IDisposable CreateCaptureEvents() - => Observable.Create(observer => + => Observable.Create(observer => { AsyncEventHandler handler = (_, args) => Capture_CapturedAsync(args, observer); this.capture.Captured += handler; - return Disposable.Create(() => this.capture.Captured -= handler); - }, rawObserver: true).Subscribe(QueueCapture); + return Disposable.Combine( + Disposable.Create(() => this.capture.Captured -= handler), + Observable.Timer(TimeSpan.Zero, TimeSpan.FromMilliseconds(100), TimeProvider.System) + .Subscribe(_ => this.trackingRequests.OnNext(default))); + }, rawObserver: true).Subscribe(_ => { }); - private async Task Capture_CapturedAsync(CapturedEventArgs args, Observer observer) + private async Task Capture_CapturedAsync(CapturedEventArgs args, Observer visibility) { - var accepted = await this.dispatcher.InvokeAsync(() => + if (this.disposedValue || visibility.IsDisposed) return; + if (this.isOneShotMode) { - if (this.disposedValue || observer.IsDisposed) return false; - if (this.isOneShotMode) this.capture.StopCapture(); - return true; - }); - if (!accepted) return; - - // フレームの寿命はイベント処理中だけなので、コピーの完了まで待つ。 - var bitmap = await SoftwareBitmap.CreateCopyFromSurfaceAsync(args.Frame.Surface); - var handedOff = false; - try - { - await this.dispatcher.InvokeAsync(() => + await Application.Current.Dispatcher.InvokeAsync(() => { - if (this.disposedValue || observer.IsDisposed) return; - this.Width = bitmap.PixelWidth; - this.Height = bitmap.PixelHeight; - if (this.disposedValue || observer.IsDisposed) return; - observer.OnNext(bitmap); - handedOff = true; + if (!this.disposedValue && !visibility.IsDisposed) this.capture.StopCapture(); }); + if (this.disposedValue || visibility.IsDisposed) return; } - finally + + // フレームの寿命はイベント処理中だけなので、コピーの完了まで待つ。 + var bitmap = await SoftwareBitmap.CreateCopyFromSurfaceAsync(args.Frame.Surface); + if (this.disposedValue || visibility.IsDisposed) { - if (!handedOff) bitmap.Dispose(); + bitmap.Dispose(); + return; } + this.Width = bitmap.PixelWidth; + this.Height = bitmap.PixelHeight; + this.captureRequests.OnNext((bitmap, visibility)); } - private async ValueTask RecognizeAsync(SoftwareBitmap capturedBitmap, CancellationToken cancellationToken) + private async ValueTask RecognizeAsync((SoftwareBitmap Bitmap, Observer Visibility) request, CancellationToken cancellationToken) { + var capturedBitmap = request.Bitmap; using var bitmap = new DisposeAction(capturedBitmap.Dispose); - var hidden = new TaskCompletionSource(); - using var visibility = ObserveHiding(hidden); - await Task.CompletedTask.ConfigureAwait(ConfigureAwaitOptions.ForceYielding); - cancellationToken.ThrowIfCancellationRequested(); - if (this.disposedValue || hidden.Task.IsCompleted) - { - return; - } + if (this.disposedValue || request.Visibility.IsDisposed) return; IEnumerable texts; using (this.Recognizing.EnterBusy()) { @@ -356,21 +290,14 @@ private async ValueTask RecognizeAsync(SoftwareBitmap capturedBitmap, Cancellati this.ocrTraceRecorder.Record(observations, new(capturedBitmap.PixelWidth, capturedBitmap.PixelHeight)); texts = observations; } - catch (ObjectDisposedException) + catch (Exception e) when (e is ObjectDisposedException or OperationCanceledException) { - // すでに破棄されている場合は何もしない - await StopRecognitionAsync(hidden.Task); - return; - } - catch (OperationCanceledException) - { - // キャンセルされた場合は何もしない - await StopRecognitionAsync(hidden.Task); + await StopCaptureAsync(request.Visibility); return; } catch (Exception e) { - if (!await StopRecognitionAsync(hidden.Task)) + if (!await StopCaptureAsync(request.Visibility)) { this.logger.LogError(e, "非表示前の画像認識でエラーが発生"); return; @@ -382,29 +309,26 @@ private async ValueTask RecognizeAsync(SoftwareBitmap capturedBitmap, Cancellati } } cancellationToken.ThrowIfCancellationRequested(); - await this.dispatcher.InvokeAsync(() => + if (this.disposedValue) { - if (this.disposedValue || hidden.Task.IsCompleted) return; - var context = new FilterContext - { - SoftwareBitmap = capturedBitmap.GetReadOnlyView(), - ImageSize = new(capturedBitmap.PixelWidth, capturedBitmap.PixelHeight), - }; - this.trackingRequests.OnNext((texts, context)); - }); + return; + } + var context = new FilterContext + { + SoftwareBitmap = capturedBitmap.GetReadOnlyView(), + ImageSize = new(capturedBitmap.PixelWidth, capturedBitmap.PixelHeight), + }; + this.trackingRequests.OnNext((texts, context, request.Visibility)); } - private async Task StopRecognitionAsync(Task hidden) - => await this.dispatcher.InvokeAsync(() => + private Task StopCaptureAsync(Observer visibility) + => Application.Current.Dispatcher.InvokeAsync(() => { - if (this.disposedValue || hidden.IsCompleted) return false; - this.captureEvents.Dispose(); - ClearPendingCapture(); - this.captureInput.Dispose(); + if (this.disposedValue || visibility.IsDisposed) return false; this.captureLoop.Dispose(); this.capture.StopCapture(); return true; - }); + }).Task; internal static IEnumerable SelectOcrTexts( IEnumerable observations, System.Drawing.Size imageSize, IOcrTextTracker tracker, bool isOneShotMode) @@ -413,21 +337,15 @@ internal static IEnumerable SelectOcrTexts( private async ValueTask?> ProcessTrackingAsync( (IEnumerable Texts, FilterContext Context) request, SoftwareBitmap bitmap, - Task hidden, + Observer visibility, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); var texts = SelectOcrTexts(request.Texts, request.Context.ImageSize, this.ocrTextTracker, this.isOneShotMode) .Select(t => t with { FontSize = t.FontSize * this.fontScale }); - if (hidden.IsCompleted) - { - return null; - } + if (visibility.IsDisposed) return null; texts = (await this.color.ConvertColorAsync(bitmap, texts)).ToArray(); - if (hidden.IsCompleted) - { - return null; - } + if (visibility.IsDisposed) return null; texts = texts.Select(t => t switch { { TranslatedText: null } when this.cache.Contains(t.SourceText) => t with { TranslatedText = this.cache.Get(t.SourceText) }, @@ -437,14 +355,13 @@ internal static IEnumerable SelectOcrTexts( return texts; } - private async ValueTask ProcessTranslationAsync((IEnumerable Texts, FilterContext Context) request, CancellationToken cancellationToken) + private async ValueTask ProcessTranslationAsync( + (IEnumerable Texts, FilterContext Context, Observer Visibility) request, + CancellationToken cancellationToken) { using var bitmap = request.Context.SoftwareBitmap; - var hidden = new TaskCompletionSource(); - using var visibility = ObserveHiding(hidden); - await Task.CompletedTask.ConfigureAwait(ConfigureAwaitOptions.ForceYielding); cancellationToken.ThrowIfCancellationRequested(); - if (this.disposedValue || hidden.Task.IsCompleted) + if (this.disposedValue || request.Visibility.IsDisposed) { return; } @@ -467,7 +384,7 @@ private async ValueTask ProcessTranslationAsync((IEnumerable Texts, Fi if (requests.Length > 0) { using var t = this.logger.LogDebugTime("翻訳"); - if (hidden.Task.IsCompleted) return; + if (request.Visibility.IsDisposed) return; var translated = await this.translator.TranslateAsync(requests).ConfigureAwait(false); this.cache.AddRange(requests.Select(t => t.SourceText).Zip(translated)); var translatedIndex = 0; @@ -487,20 +404,12 @@ private async ValueTask ProcessTranslationAsync((IEnumerable Texts, Fi catch (Exception e) { this.logger.LogError(e, "翻訳中にエラーが発生"); - var stopped = await this.dispatcher.InvokeAsync(() => + if (await StopCaptureAsync(request.Visibility)) { - if (hidden.Task.IsCompleted || this.disposedValue) return false; - this.captureEvents.Dispose(); - ClearPendingCapture(); - this.captureInput.Dispose(); - this.captureLoop.Dispose(); - this.trackingTimer.Stop(); this.trackingLoop.Dispose(); this.translationLoop.Dispose(); - this.capture.StopCapture(); - return true; - }); - if (stopped) await ShowErrorAsync(Resources.FaildOverlay, e, string.Empty); + await ShowErrorAsync(Resources.FaildOverlay, e, string.Empty); + } return; } using (this.Filtering.EnterBusy()) @@ -514,42 +423,26 @@ private async ValueTask ProcessTranslationAsync((IEnumerable Texts, Fi texts = await tmp.ToArrayAsync(cancellationToken); } cancellationToken.ThrowIfCancellationRequested(); - await this.dispatcher.InvokeAsync(() => - { - if (!this.disposedValue) - { - UpdateOcrTexts(texts, hidden.Task); - } - }); + this.uiRequests.OnNext((texts, request.Visibility)); } - private void UpdateOcrTexts(IEnumerable texts, Task hidden) + private void UpdateOcrTexts(IEnumerable texts) { this.logger.LogDebug("UI適用"); var hash = texts.Select(t => t with { Background = Color.FromArgb((int)(255 * this.overlayOpacity), t.Background) }).ToHashSet(); - this.updatingOcrTexts = true; - try + foreach (var text in this.OcrTexts.Where(t => !hash.Contains(t)).ToArray()) { - foreach (var text in this.OcrTexts.Where(t => !hash.Contains(t)).ToArray()) - { - if (hidden.IsCompleted) return; - this.OcrTexts.Remove(text); - } - hash.ExceptWith(this.OcrTexts); - foreach (var text in hash) - { - if (hidden.IsCompleted) return; - this.OcrTexts.Add(text); - } + this.OcrTexts.Remove(text); } - finally + hash.ExceptWith(this.OcrTexts); + foreach (var text in hash) { - this.updatingOcrTexts = false; + this.OcrTexts.Add(text); } } private Task ShowErrorAsync(string message, Exception exception, string path) - => this.dispatcher.InvokeAsync(async () => + => Application.Current.Dispatcher.InvokeAsync(async () => { await this.presentationService.OpenErrorDialogAsync(message, exception, this.name, path); StrongReferenceMessenger.Default.Send(new(this)); @@ -565,20 +458,17 @@ protected virtual void Dispose(bool disposing) if (disposing) { // 実行中の処理から遅れて通知されても受け付けない。 - this.captureEvents.Dispose(); - ClearPendingCapture(); + this.captureLoop.Dispose(); this.captureRequests.OnCompleted(); - this.recognitionRequests.OnCompleted(); this.translationRequests.OnCompleted(); + this.uiRequests.OnCompleted(); if (this.capture is IDisposable captureDisposable) { captureDisposable.Dispose(); } - this.captureInput.Dispose(); - this.captureLoop.Dispose(); - this.trackingTimer.Stop(); this.trackingLoop.Dispose(); this.translationLoop.Dispose(); + this.uiLoop.Dispose(); } } diff --git a/WindowTranslator/Modules/Main/OverlayMainWindow.xaml b/WindowTranslator/Modules/Main/OverlayMainWindow.xaml index eda07005..3bd46f83 100644 --- a/WindowTranslator/Modules/Main/OverlayMainWindow.xaml +++ b/WindowTranslator/Modules/Main/OverlayMainWindow.xaml @@ -15,6 +15,7 @@ d:Width="800" AllowsTransparency="True" Background="Transparent" + DataContextChanged="OverlayVisibilityChanged" Loaded="Window_Loaded" ResizeMode="NoResize" ShowInTaskbar="False" @@ -24,20 +25,19 @@ - + Texts="{Binding OcrTexts}" /> Date: Sun, 8 Nov 2026 17:07:42 +0900 Subject: [PATCH 8/8] =?UTF-8?q?=E9=9D=9E=E8=A1=A8=E7=A4=BA=E4=B8=AD?= =?UTF-8?q?=E3=81=AE=E8=BF=BD=E8=B7=A1=E5=81=9C=E6=AD=A2=E3=82=92=E5=BF=85?= =?UTF-8?q?=E8=A6=81=E6=9C=80=E5=B0=8F=E9=99=90=E3=81=AE=E5=A4=89=E6=9B=B4?= =?UTF-8?q?=E3=81=AB=E7=B5=9E=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 表示期間ごとのキャプチャ購読を既存の表示状態の購読から作り、非表示で破棄する。 この購読を OCR・追跡・翻訳に渡し、破棄済みの購読から来た結果は追跡・翻訳要求・表示に使わない。 フィールドの追加、非表示時の保持画像の破棄、各段階の個別判定などを取り除いた。 Co-Authored-By: Claude Opus 5.5 --- .../OverlayVisibilityTests.cs | 234 ++++++++++++------ .../Modules/Main/MainViewModelBase.cs | 166 +++++-------- .../Modules/Main/OverlayMainWindow.xaml | 1 + 3 files changed, 213 insertions(+), 188 deletions(-) diff --git a/WindowTranslator.Tests/OverlayVisibilityTests.cs b/WindowTranslator.Tests/OverlayVisibilityTests.cs index 590f3f4f..e5b8c6a7 100644 --- a/WindowTranslator.Tests/OverlayVisibilityTests.cs +++ b/WindowTranslator.Tests/OverlayVisibilityTests.cs @@ -82,9 +82,9 @@ private static async Task RunInStaAsync() await VerifyInFlightAsync(stage); } await VerifyInFlightAsync(Stage.Ocr, withoutContext: true); - await VerifyInFlightAsync(Stage.Ocr, failFirst: true); - await VerifyInFlightAsync(Stage.Translation, failFirst: true); await VerifyQueuedCapturesAsync(); + await VerifyQueuedOcrWhileHiddenAsync(); + await VerifyOcrStopsAfterCancellationAsync(); await VerifyOcrCompletesWhileHiddenAsync(); await VerifyHiddenDuringResetAsync(); await VerifyHiddenDuringTrackingAsync(); @@ -100,7 +100,6 @@ private static async Task RunInStaAsync() await VerifyOneShotWindowVisibilityAsync(); await VerifyDelayedCaptureAsync(isOneShotMode: false); await VerifyDelayedCaptureAsync(isOneShotMode: true); - await VerifyQueuedOneShotStopAsync(); await VerifyDisposeAsync(); } catch (Exception exception) @@ -129,36 +128,29 @@ private static async Task RunInStaAsync() private static async Task VerifyHiddenAsync() { await using var fixture = new Fixture(); - SoftwareBitmap? tracked = null; - var hiddenTicks = 0; var hiddenTranslations = 0; - using var tracking = fixture.Tracking.Subscribe(request => - { - if (request.Context is { } context) Volatile.Write(ref tracked, context.SoftwareBitmap); - else if (!fixture.Model.OverlayVisible) - { - Interlocked.Increment(ref hiddenTicks); - } - }); using var translations = fixture.Translations.Subscribe(request => { - if (request is not null && !fixture.Model.OverlayVisible) Interlocked.Increment(ref hiddenTranslations); + if (!fixture.Model.OverlayVisible) Interlocked.Increment(ref hiddenTranslations); }); fixture.SubmitCapture(Bitmap(320)); - await WaitUntilAsync(() => fixture.Translator.Gate.Count > 0 && Volatile.Read(ref tracked) is not null); + await WaitUntilAsync(() => fixture.Translator.Gate.Count > 0); + var trackerCountBeforeHidden = fixture.Tracker.UpdateCount; + await Task.Delay(250); + // 表示中は追跡周期ごとに追跡が続いていることを確認してから非表示にする。 + Assert.True(fixture.Tracker.UpdateCount > trackerCountBeforeHidden); fixture.Model.OverlayVisible = false; + // 非表示の直前に始まった追跡は終わるまで待つ。 + await Task.Delay(150); var trackerCount = fixture.Tracker.UpdateCount; var colorCount = fixture.Color.Gate.Count; - await WaitUntilAsync(() => IsDisposed(Volatile.Read(ref tracked)!)); - await Task.Delay(150); - var ticksAfterStopping = Volatile.Read(ref hiddenTicks); - await Task.Delay(250); + var translationCount = fixture.Translator.Gate.Count; + await Task.Delay(400); - Assert.Equal(ticksAfterStopping, Volatile.Read(ref hiddenTicks)); Assert.Equal(trackerCount, fixture.Tracker.UpdateCount); Assert.Equal(colorCount, fixture.Color.Gate.Count); + Assert.Equal(translationCount, fixture.Translator.Gate.Count); Assert.Equal(0, Volatile.Read(ref hiddenTranslations)); - Assert.True(IsDisposed(Volatile.Read(ref tracked)!)); fixture.Model.OverlayVisible = true; await System.Windows.Threading.Dispatcher.Yield(System.Windows.Threading.DispatcherPriority.Background); Assert.Empty(fixture.Model.OcrTexts); @@ -169,9 +161,9 @@ private static async Task VerifyHiddenAsync() Assert.True(fixture.Tracker.ResetCount >= 2); } - private static async Task VerifyInFlightAsync(Stage stage, bool withoutContext = false, bool failFirst = false) + private static async Task VerifyInFlightAsync(Stage stage, bool withoutContext = false) { - await using var fixture = new Fixture(stage, withoutContext: withoutContext, failFirst: failFirst); + await using var fixture = new Fixture(stage, withoutContext: withoutContext); var gate = stage switch { Stage.Ocr => fixture.Ocr.Gate, @@ -219,9 +211,10 @@ private static async Task VerifyQueuedCapturesAsync() throttled.Model.OverlayVisible = false; throttled.Model.OverlayVisible = true; await WaitUntilAsync(() => IsDisposed(old)); - Assert.False(throttled.Ocr.Gate.Started.Task.IsCompleted); + Assert.DoesNotContain(throttled.Model.OcrTexts, text => text.SourceText == "old"); throttled.SubmitCapture(Bitmap(321)); await WaitUntilAsync(() => throttled.Model.OcrTexts.Any(text => text.SourceText == "new")); + Assert.DoesNotContain(throttled.Model.OcrTexts, text => text.SourceText == "old"); } await using (var queued = new Fixture(Stage.Ocr)) @@ -235,35 +228,85 @@ private static async Task VerifyQueuedCapturesAsync() queued.Model.OverlayVisible = true; queued.Ocr.Gate.Release.TrySetResult(); await WaitUntilAsync(() => IsDisposed(old)); - Assert.False(queued.Ocr.Gate.NextStarted.Task.IsCompleted); + Assert.DoesNotContain(queued.Model.OcrTexts, text => text.SourceText == "old"); queued.SubmitCapture(Bitmap(321)); await WaitUntilAsync(() => queued.Model.OcrTexts.Any(text => text.SourceText == "new")); + Assert.DoesNotContain(queued.Model.OcrTexts, text => text.SourceText == "old"); + } + } + + private static async Task VerifyQueuedOcrWhileHiddenAsync() + { + await using var fixture = new Fixture(Stage.Ocr); + fixture.SubmitCapture(Bitmap(320)); + await fixture.Ocr.Gate.Started.Task.WaitAsync(TimeSpan.FromSeconds(5)); + var pending = Bitmap(322); + fixture.SubmitCapture(pending); + await Task.Delay(150); + fixture.Model.OverlayVisible = false; + fixture.Ocr.Gate.Release.TrySetResult(); + await fixture.Ocr.Gate.NextStarted.Task.WaitAsync(TimeSpan.FromSeconds(5)); + await WaitUntilAsync(() => IsDisposed(pending)); + Assert.Equal(0, fixture.Tracker.UpdateCount); + Assert.Equal(0, fixture.Color.Gate.Count); + Assert.Equal(0, fixture.Translator.Gate.Count); + + fixture.Model.OverlayVisible = true; + fixture.SubmitCapture(Bitmap(321)); + await WaitUntilAsync(() => fixture.Model.OcrTexts.Any(text => text.SourceText == "new")); + Assert.DoesNotContain(fixture.Model.OcrTexts, text => text.SourceText == "old"); + } + + private static async Task VerifyOcrStopsAfterCancellationAsync() + { + await using var fixture = new Fixture(Stage.Ocr); + fixture.Ocr.CancelFirst = true; + fixture.SubmitCapture(Bitmap(320)); + await fixture.Ocr.Gate.Started.Task.WaitAsync(TimeSpan.FromSeconds(5)); + var pending = Bitmap(322); + try + { + fixture.SubmitCapture(pending); + await Task.Delay(150); + fixture.Ocr.Gate.Release.TrySetResult(); + await WaitUntilAsync(() => fixture.Capture.StopCount > 0); + await Task.Delay(250); + Assert.False(fixture.Ocr.Gate.NextStarted.Task.IsCompleted); + } + finally + { + fixture.Model.Dispose(); + if (!IsDisposed(pending)) pending.Dispose(); } } private static async Task VerifyHiddenDuringTrackingAsync() { await using var fixture = new Fixture(delayUpdate: true); + var hiddenTranslation = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + using var translations = fixture.Translations.Subscribe(request => + { + if (!fixture.Model.OverlayVisible) hiddenTranslation.TrySetResult(); + }); fixture.SubmitCapture(Bitmap(320)); await fixture.Tracker.Updated.Task.WaitAsync(TimeSpan.FromSeconds(5)); fixture.Model.OverlayVisible = false; fixture.Tracker.UpdateRelease.Set(); await Task.Delay(250); - Assert.Equal(0, fixture.Color.Gate.Count); + // 非表示前に始まった追跡の残りは許容するが、翻訳要求は送らない。 + Assert.Equal(1, fixture.Tracker.UpdateCount); + Assert.False(hiddenTranslation.Task.IsCompleted); + Assert.Equal(0, fixture.Translator.Gate.Count); fixture.Model.OverlayVisible = true; fixture.SubmitCapture(Bitmap(321)); await WaitUntilAsync(() => fixture.Model.OcrTexts.Any(text => text.SourceText == "new")); + Assert.DoesNotContain(fixture.Model.OcrTexts, text => text.SourceText == "old"); } private static async Task VerifyOcrCompletesWhileHiddenAsync() { await using var fixture = new Fixture(Stage.Ocr); - SoftwareBitmap? tracked = null; - using var tracking = fixture.Tracking.Subscribe(request => - { - if (request.Context is { } context) tracked = context.SoftwareBitmap; - }); var image = Bitmap(320); fixture.SubmitCapture(image); await fixture.Ocr.Gate.Started.Task.WaitAsync(TimeSpan.FromSeconds(5)); @@ -272,7 +315,6 @@ private static async Task VerifyOcrCompletesWhileHiddenAsync() fixture.Ocr.Gate.Release.TrySetResult(); await WaitUntilAsync(() => IsDisposed(image)); await Task.Delay(200); - Assert.True(tracked is null || IsDisposed(tracked)); Assert.Equal(0, fixture.Tracker.UpdateCount); Assert.Equal(0, fixture.Color.Gate.Count); Assert.Equal(0, fixture.Translator.Gate.Count); @@ -313,7 +355,7 @@ private static async Task VerifyHiddenDuringColorAsync() fixture.Model.OverlayVisible = false; fixture.Color.Gate.Release.TrySetResult(); await WaitUntilAsync(() => fixture.Color.Bitmap is { } bitmap && IsDisposed(bitmap)); - Assert.Equal(0, fixture.Cache.ContainsCount); + Assert.Equal(0, fixture.Translator.Gate.Count); } private static async Task VerifyUiResponsiveDuringTranslationErrorAsync() @@ -351,7 +393,7 @@ private static async Task VerifyHiddenDuringCacheAsync() var hiddenTranslation = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); using var translations = fixture.Translations.Subscribe(request => { - if (request is not null && !fixture.Model.OverlayVisible) hiddenTranslation.TrySetResult(); + if (!fixture.Model.OverlayVisible) hiddenTranslation.TrySetResult(); }); fixture.SubmitCapture(Bitmap(320)); await WaitUntilAsync(() => fixture.Model.OcrTexts.Any(text => text.SourceText == "old")); @@ -368,22 +410,29 @@ private static async Task VerifyHiddenDuringCacheAsync() private static async Task VerifyOldHandoffAfterReopenAsync() { - await using var fixture = new Fixture(); - fixture.Ocr.Text = "handoff-old"; - fixture.SubmitCapture(Bitmap(320)); - await WaitUntilAsync(() => fixture.Capture.StopCount > 0 - || fixture.Model.OcrTexts.Any(text => text.SourceText == "handoff-old")); - var bitmap = fixture.Color.Bitmap!; - if (fixture.Capture.StopCount == 0) + await using var fixture = new Fixture(Stage.Translation); + try { - fixture.Model.OverlayVisible = false; - fixture.Model.OverlayVisible = true; + fixture.Ocr.Text = "handoff-old"; + fixture.SubmitCapture(Bitmap(320)); + await WaitUntilAsync(() => fixture.Capture.StopCount > 0 + || fixture.Model.OcrTexts.Any(text => text.SourceText == "handoff-old")); + var bitmap = fixture.Color.Bitmap!; + if (fixture.Capture.StopCount == 0) + { + fixture.Model.OverlayVisible = false; + fixture.Model.OverlayVisible = true; + } + await WaitUntilAsync(() => IsDisposed(bitmap)); + fixture.Ocr.Text = "new"; + fixture.SubmitCapture(Bitmap(321)); + await WaitUntilAsync(() => fixture.Model.OcrTexts.Any(text => text.SourceText == "new")); + Assert.DoesNotContain(fixture.Model.OcrTexts, text => text.SourceText == "handoff-old"); + } + finally + { + fixture.Translator.Gate.Release.TrySetResult(); } - await WaitUntilAsync(() => IsDisposed(bitmap)); - fixture.Ocr.Text = "new"; - fixture.SubmitCapture(Bitmap(321)); - await WaitUntilAsync(() => fixture.Model.OcrTexts.Any(text => text.SourceText == "new")); - Assert.DoesNotContain(fixture.Model.OcrTexts, text => text.SourceText == "handoff-old"); } private static async Task VerifyHiddenBeforeTranslationAsync() @@ -395,15 +444,22 @@ private static async Task VerifyHiddenBeforeTranslationAsync() fixture.Translations.OnNext(([new("pending", 0, 0, 100, 30, 20, false)], new() { SoftwareBitmap = pending, ImageSize = new(322, 240) }, fixture.Visibility)); fixture.Model.OverlayVisible = false; - Assert.True(IsDisposed(pending)); fixture.Filter.Gate.Release.TrySetResult(); + await WaitUntilAsync(() => IsDisposed(pending)); await Task.Delay(250); - Assert.False(fixture.Translator.Gate.Started.Task.IsCompleted); + // 非表示前に始まった翻訳の残りだけが実行され、待機していた要求は翻訳しない。 + Assert.Equal(1, fixture.Translator.Gate.Count); + Assert.Equal(1, fixture.Filter.Gate.Count); + var shown = new List(); + fixture.Model.OcrTexts.CollectionChanged += (_, _) => + shown.AddRange(fixture.Model.OcrTexts.Select(text => text.SourceText)); fixture.Model.OverlayVisible = true; fixture.SubmitCapture(Bitmap(321)); - await WaitUntilAsync(() => fixture.Translator.Gate.Started.Task.IsCompleted); + await WaitUntilAsync(() => fixture.Translator.Gate.Count > 1); await WaitUntilAsync(() => fixture.Model.OcrTexts.Any(text => text.SourceText == "new")); + Assert.DoesNotContain("old", shown); + Assert.DoesNotContain("pending", shown); } private static async Task VerifyOldEmptyResultDoesNotRemoveNewAsync() @@ -431,38 +487,31 @@ private static async Task VerifyDelayedCaptureAsync(bool isOneShotMode) { await using var fixture = new Fixture(isOneShotMode: isOneShotMode); var handler = fixture.Capture.CurrentHandler!; + var visibility = fixture.Visibility; fixture.Model.OverlayVisible = false; Assert.Null(fixture.Capture.CurrentHandler); + Assert.True(visibility.IsDisposed); fixture.Model.OverlayVisible = true; Assert.NotNull(fixture.Capture.CurrentHandler); Assert.NotSame(handler, fixture.Capture.CurrentHandler); - var stops = fixture.Capture.StopCount; - var error = await Record.ExceptionAsync(() => handler.InvokeAsync(fixture.Capture, null!)); - await Task.Delay(150); - Assert.Equal(stops, fixture.Capture.StopCount); - Assert.Null(error); - Assert.False(fixture.Ocr.Gate.Started.Task.IsCompleted); - } + Assert.False(fixture.Visibility.IsDisposed); - private static async Task VerifyQueuedOneShotStopAsync() - { - await using var fixture = new Fixture(isOneShotMode: true); - var handler = fixture.Capture.CurrentHandler!; - using var submitted = new ManualResetEventSlim(); - var delayedCapture = Task.Run(() => - { - var operation = handler.InvokeAsync(fixture.Capture, null!); - submitted.Set(); - return operation; - }); - Assert.True(submitted.Wait(TimeSpan.FromSeconds(5))); - fixture.Model.OverlayVisible = false; - fixture.Model.OverlayVisible = true; - var stops = fixture.Capture.StopCount; - await delayedCapture.WaitAsync(TimeSpan.FromSeconds(5)); - Assert.Equal(stops, fixture.Capture.StopCount); + // 非表示前に取得した画像の認識結果は、再表示後も表示しない。 + var shown = new List(); + fixture.Model.OcrTexts.CollectionChanged += (_, _) => + shown.AddRange(fixture.Model.OcrTexts.Select(text => text.SourceText)); + Field)>>(fixture.Model, "captureRequests").OnNext((Bitmap(320), visibility)); + await WaitUntilAsync(() => fixture.Ocr.Gate.Count == 1); + await Task.Delay(250); + Assert.Equal(0, fixture.Translator.Gate.Count); + fixture.SubmitCapture(Bitmap(321)); + await WaitUntilAsync(() => fixture.Model.OcrTexts.Any(text => text.SourceText == "new")); + Assert.DoesNotContain("old", shown); } + private static T Field(MainViewModelBase model, string name) where T : class + => (T)typeof(MainViewModelBase).GetField(name, BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(model)!; + private static async Task VerifyOneShotAsync() { await using var fixture = new Fixture(Stage.Translation, captureInterval: 10, isOneShotMode: true); @@ -620,6 +669,19 @@ private static void StopPositionTimer(OverlayMainWindow window) private static async Task VerifyDisposeAsync() { + await using (var trackedFixture = new Fixture()) + { + SoftwareBitmap? tracked = null; + using var tracking = trackedFixture.Tracking.Subscribe(request => + { + if (request.Context is { } context) Volatile.Write(ref tracked, context.SoftwareBitmap); + }); + trackedFixture.SubmitCapture(Bitmap(320)); + await WaitUntilAsync(() => Volatile.Read(ref tracked) is not null); + trackedFixture.Model.Dispose(); + await WaitUntilAsync(() => IsDisposed(Volatile.Read(ref tracked)!)); + } + await using var fixture = new Fixture(Stage.Ocr); var image = Bitmap(320); fixture.SubmitCapture(image); @@ -694,11 +756,20 @@ public Fixture(Stage? delayed = null, double captureInterval = 0, bool delayRese public Filter Filter { get; } public PresentationProxy Presentation { get; } public OverlayMainViewModel Model { get; } - public Subject<(IEnumerable? Texts, FilterContext? Context, Observer? Visibility)> Tracking - => Field?, FilterContext?, Observer?)>>("trackingRequests"); - public Subject<(IEnumerable Texts, FilterContext Context, Observer Visibility)?> Translations - => Field, FilterContext, Observer)?>>("translationRequests"); - public Observer Visibility => (Observer)Field("captureLoop").Disposable!; + public Subject<(IEnumerable Texts, FilterContext Context, Observer Visibility)> Tracking + => Field, FilterContext, Observer)>>("trackingRequests"); + public Subject<(IEnumerable Texts, FilterContext Context, Observer Visibility)> Translations + => Field, FilterContext, Observer)>>("translationRequests"); + + // 表示期間の購読は、登録中のキャプチャハンドラーが保持している。 + public Observer Visibility + { + get + { + var target = this.Capture.CurrentHandler!.Target!; + return (Observer)target.GetType().GetFields().Single(f => f.FieldType == typeof(Observer)).GetValue(target)!; + } + } public void SubmitCapture(SoftwareBitmap bitmap) => Field)>>("captureRequests").OnNext((bitmap, this.Visibility)); @@ -768,10 +839,12 @@ public void StartCapture(IntPtr targetWindow) { } private sealed class Ocr(bool delayed, bool failFirst, bool emptyFirstCapture) : IOcrModule { public Gate Gate { get; } = new(delayed); + public bool CancelFirst { get; set; } public string? Text { get; set; } public async ValueTask> RecognizeAsync(OcrCaptureInput input) { await this.Gate.EnterAsync(); + if (this.CancelFirst && this.Gate.Count == 1) throw new OperationCanceledException(); if (failFirst && this.Gate.Count == 1) throw new InvalidOperationException("模擬 OCR エラー"); if (emptyFirstCapture && input.Source.PixelWidth == 320) return []; var text = this.Text ?? (input.Source.PixelWidth == 321 ? "new" : "old"); @@ -877,7 +950,6 @@ public async IAsyncEnumerable ExecutePreTranslate(IAsyncEnumerable Volatile.Read(ref this.containsCount); public TaskCompletionSource Started { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); public ManualResetEventSlim Release { get; } = new(); public bool Contains(string src) diff --git a/WindowTranslator/Modules/Main/MainViewModelBase.cs b/WindowTranslator/Modules/Main/MainViewModelBase.cs index c4e60135..8ad9ca18 100644 --- a/WindowTranslator/Modules/Main/MainViewModelBase.cs +++ b/WindowTranslator/Modules/Main/MainViewModelBase.cs @@ -24,10 +24,10 @@ namespace WindowTranslator.Modules.Main; public abstract partial class MainViewModelBase : IDisposable { private readonly Subject<(SoftwareBitmap Bitmap, Observer Visibility)> captureRequests = new(); - private readonly Subject<(IEnumerable? Texts, FilterContext? Context, Observer? Visibility)> trackingRequests = new(); - private readonly Subject<(IEnumerable Texts, FilterContext Context, Observer Visibility)?> translationRequests = new(); - private readonly Subject<(IEnumerable Texts, Observer? Visibility)> uiRequests = new(); - private readonly SerialDisposable captureLoop = new(); + private readonly Subject<(IEnumerable Texts, FilterContext Context, Observer Visibility)> trackingRequests = new(); + private readonly Subject<(IEnumerable Texts, FilterContext Context, Observer Visibility)> translationRequests = new(); + private readonly Subject> uiRequests = new(); + private readonly IDisposable captureLoop; private readonly IDisposable trackingLoop; private readonly IDisposable translationLoop; private readonly IDisposable uiLoop; @@ -65,7 +65,7 @@ public abstract partial class MainViewModelBase : IDisposable public BusyScope Recognizing { get; } = new(); public BusyScope Filtering { get; } = new(); - private volatile bool isFirstCapture = true; + private bool isFirstCapture = true; private volatile bool disposedValue; public ObservableCollection OcrTexts { get; } = []; @@ -108,82 +108,55 @@ public MainViewModelBase( this.logger = logger; this.uiLoop = this.uiRequests .ObserveOn(new DispatcherSynchronizationContext(Application.Current.Dispatcher)) - .Where(request => request.Visibility?.IsDisposed != true) - .Subscribe(request => UpdateOcrTexts(request.Texts)); + .Where(_ => this.OverlayVisible) + .Subscribe(UpdateOcrTexts); var captureInterval = TimeSpan.FromSeconds(options.Value.CaptureInterval); var captures = !this.isOneShotMode && captureInterval > TimeSpan.Zero ? this.captureRequests.ThrottleLast(captureInterval) : this.captureRequests; + this.captureLoop = captures + .ObserveOnThreadPool() + .SubscribeAwait((request, ct) => RecognizeAsync(request.Bitmap, request.Visibility, ct), AwaitOperation.ThrottleFirstLast, configureAwait: false); this.trackingLoop = Disposable.Combine( Disposable.Create(() => this.trackingRequests.OnCompleted()), - captures.ObserveOnThreadPool() - .SubscribeAwait(RecognizeAsync, AwaitOperation.ThrottleFirstLast, configureAwait: false), + this.ObservePropertyChanged(x => x.OverlayVisible).Where(v => v).Subscribe(_ => SubscribeWhileVisible()), + Observable.Timer(TimeSpan.Zero, TimeSpan.FromMilliseconds(100), TimeProvider.System).Subscribe(_ => this.trackingRequests.OnNext(default)), this.trackingRequests .Materialize() .Synchronize() // タイマーの空通知では直前のOCR結果を再利用する。 - .Scan(default((IEnumerable? Texts, FilterContext? Context, Observer? Visibility)), (previous, current) => + .Scan(default((IEnumerable Texts, FilterContext Context, Observer Visibility)), (previous, current) => { - if (current.Kind != NotificationKind.OnNext || previous.Visibility?.IsDisposed == true) - { - previous.Context?.SoftwareBitmap.Dispose(); - previous = default; - } - if (current.Kind != NotificationKind.OnNext) - { - return default; - } if (current.Value.Context is { } context) { - if (current.Value.Visibility!.IsDisposed) - { - context.SoftwareBitmap.Dispose(); - return previous; - } - if (previous.Context is { } previousContext && previousContext.ImageSize != context.ImageSize) + if (previous.Context?.ImageSize != context.ImageSize || previous.Visibility != current.Value.Visibility) { this.ocrTextTracker.Reset(); } - if (this.isOneShotMode) - { - this.isFirstCapture = true; - } + this.isFirstCapture |= this.isOneShotMode; } - else + else if (current.Kind == NotificationKind.OnNext) { return previous; } previous.Context?.SoftwareBitmap.Dispose(); return current.Value; }) - .Where(request => request.Context is not null) + .Where(request => request.Context is not null && !request.Visibility.IsDisposed) .SubscribeAwait(async (request, ct) => { - using var bitmap = request.Context!.SoftwareBitmap.GetReadOnlyView(); - ct.ThrowIfCancellationRequested(); - if (this.isFirstCapture) this.ocrTextTracker.Reset(); - if (request.Visibility!.IsDisposed) return; - var texts = await ProcessTrackingAsync((request.Texts!, request.Context), bitmap, request.Visibility, ct); - if (texts is null) return; - if (request.Visibility.IsDisposed || this.disposedValue) return; + using var bitmap = request.Context.SoftwareBitmap.GetReadOnlyView(); + var texts = await ProcessTrackingAsync((request.Texts, request.Context), bitmap, ct); + if (request.Visibility.IsDisposed) return; if (this.isFirstCapture) { this.isFirstCapture = false; - if (request.Visibility.IsDisposed) - { - // 非表示前の処理で再表示後の初回フラグを消さない。 - this.isFirstCapture = true; - return; - } - this.uiRequests.OnNext((texts, request.Visibility)); + this.uiRequests.OnNext(texts); } this.translationRequests.OnNext((texts, request.Context with { SoftwareBitmap = bitmap.GetReadOnlyView(), }, request.Visibility)); }, AwaitOperation.Drop, configureAwait: false, cancelOnCompleted: true)); this.translationLoop = this.translationRequests - .SubscribeAwaitThrottleFirstLast( - (request, ct) => request is { } value ? ProcessTranslationAsync(value, ct) : ValueTask.CompletedTask, - static request => request?.Context.SoftwareBitmap.Dispose(), configureAwait: false); - this.captureLoop.Disposable = CreateCaptureEvents(); + .SubscribeAwaitThrottleFirstLast(ProcessTranslationAsync, static r => r.Context.SoftwareBitmap.Dispose(), configureAwait: false); if (!this.isOneShotMode) { this.capture.StartCapture(this.processInfoStore.MainWindowHandle); @@ -196,47 +169,40 @@ partial void OnOverlayVisibleChanged(bool value) { if (value) { - this.uiRequests.OnNext(([], null)); + this.uiRequests.OnNext([]); this.isFirstCapture = !this.isOneShotMode; - this.captureLoop.Disposable = CreateCaptureEvents(); + // Start capture when overlay becomes visible this.capture.StartCapture(this.processInfoStore.MainWindowHandle); } else { - this.captureLoop.Disposable = null; - this.translationRequests.OnNext(null); - ThreadPool.QueueUserWorkItem(static model => model.trackingRequests.OnNext(default), this, preferLocal: false); + // Stop capture when overlay becomes hidden this.capture.StopCapture(); this.logger.LogDebug("Overlay became hidden - capture stopped"); } } - private IDisposable CreateCaptureEvents() + // 表示中だけキャプチャを受け付け、非表示で購読を破棄して表示前の処理結果を見分ける。 + private void SubscribeWhileVisible() => Observable.Create(observer => { AsyncEventHandler handler = (_, args) => Capture_CapturedAsync(args, observer); this.capture.Captured += handler; - return Disposable.Combine( - Disposable.Create(() => this.capture.Captured -= handler), - Observable.Timer(TimeSpan.Zero, TimeSpan.FromMilliseconds(100), TimeProvider.System) - .Subscribe(_ => this.trackingRequests.OnNext(default))); - }, rawObserver: true).Subscribe(_ => { }); + return Disposable.Create(() => this.capture.Captured -= handler); + }, rawObserver: true) + .TakeUntil(this.ObservePropertyChanged(x => x.OverlayVisible, pushCurrentValueOnSubscribe: false).Where(v => !v)) + .Subscribe(); private async Task Capture_CapturedAsync(CapturedEventArgs args, Observer visibility) { - if (this.disposedValue || visibility.IsDisposed) return; if (this.isOneShotMode) { - await Application.Current.Dispatcher.InvokeAsync(() => - { - if (!this.disposedValue && !visibility.IsDisposed) this.capture.StopCapture(); - }); - if (this.disposedValue || visibility.IsDisposed) return; + this.capture.StopCapture(); } // フレームの寿命はイベント処理中だけなので、コピーの完了まで待つ。 var bitmap = await SoftwareBitmap.CreateCopyFromSurfaceAsync(args.Frame.Surface); - if (this.disposedValue || visibility.IsDisposed) + if (this.disposedValue) { bitmap.Dispose(); return; @@ -246,11 +212,9 @@ await Application.Current.Dispatcher.InvokeAsync(() => this.captureRequests.OnNext((bitmap, visibility)); } - private async ValueTask RecognizeAsync((SoftwareBitmap Bitmap, Observer Visibility) request, CancellationToken cancellationToken) + private async ValueTask RecognizeAsync(SoftwareBitmap capturedBitmap, Observer visibility, CancellationToken cancellationToken) { - var capturedBitmap = request.Bitmap; using var bitmap = new DisposeAction(capturedBitmap.Dispose); - if (this.disposedValue || request.Visibility.IsDisposed) return; IEnumerable texts; using (this.Recognizing.EnterBusy()) { @@ -290,18 +254,24 @@ private async ValueTask RecognizeAsync((SoftwareBitmap Bitmap, Observer Vi this.ocrTraceRecorder.Record(observations, new(capturedBitmap.PixelWidth, capturedBitmap.PixelHeight)); texts = observations; } - catch (Exception e) when (e is ObjectDisposedException or OperationCanceledException) + catch (ObjectDisposedException) { - await StopCaptureAsync(request.Visibility); + // すでに破棄されている場合は何もしない + this.captureLoop.Dispose(); + await Application.Current.Dispatcher.InvokeAsync(this.capture.StopCapture); + return; + } + catch (OperationCanceledException) + { + // キャンセルされた場合は何もしない + this.captureLoop.Dispose(); + await Application.Current.Dispatcher.InvokeAsync(this.capture.StopCapture); return; } catch (Exception e) { - if (!await StopCaptureAsync(request.Visibility)) - { - this.logger.LogError(e, "非表示前の画像認識でエラーが発生"); - return; - } + this.captureLoop.Dispose(); + await Application.Current.Dispatcher.InvokeAsync(this.capture.StopCapture); var path = Path.Combine(PathUtility.UserDir, $"ocr_error", $"{DateTime.UtcNow:yyyyMMdd'T'HHmmss'Z'}.png"); await capturedBitmap.TrySaveImage(path); await ShowErrorAsync(Resources.FaildOcr, e, path); @@ -318,34 +288,19 @@ private async ValueTask RecognizeAsync((SoftwareBitmap Bitmap, Observer Vi SoftwareBitmap = capturedBitmap.GetReadOnlyView(), ImageSize = new(capturedBitmap.PixelWidth, capturedBitmap.PixelHeight), }; - this.trackingRequests.OnNext((texts, context, request.Visibility)); + this.trackingRequests.OnNext((texts, context, visibility)); } - private Task StopCaptureAsync(Observer visibility) - => Application.Current.Dispatcher.InvokeAsync(() => - { - if (this.disposedValue || visibility.IsDisposed) return false; - this.captureLoop.Dispose(); - this.capture.StopCapture(); - return true; - }).Task; - internal static IEnumerable SelectOcrTexts( IEnumerable observations, System.Drawing.Size imageSize, IOcrTextTracker tracker, bool isOneShotMode) => isOneShotMode ? observations : tracker.Update(observations, imageSize); - private async ValueTask?> ProcessTrackingAsync( - (IEnumerable Texts, FilterContext Context) request, - SoftwareBitmap bitmap, - Observer visibility, - CancellationToken cancellationToken) + private async ValueTask> ProcessTrackingAsync((IEnumerable Texts, FilterContext Context) request, SoftwareBitmap bitmap, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); var texts = SelectOcrTexts(request.Texts, request.Context.ImageSize, this.ocrTextTracker, this.isOneShotMode) .Select(t => t with { FontSize = t.FontSize * this.fontScale }); - if (visibility.IsDisposed) return null; texts = (await this.color.ConvertColorAsync(bitmap, texts)).ToArray(); - if (visibility.IsDisposed) return null; texts = texts.Select(t => t switch { { TranslatedText: null } when this.cache.Contains(t.SourceText) => t with { TranslatedText = this.cache.Get(t.SourceText) }, @@ -355,13 +310,11 @@ internal static IEnumerable SelectOcrTexts( return texts; } - private async ValueTask ProcessTranslationAsync( - (IEnumerable Texts, FilterContext Context, Observer Visibility) request, - CancellationToken cancellationToken) + private async ValueTask ProcessTranslationAsync((IEnumerable Texts, FilterContext Context, Observer Visibility) request, CancellationToken cancellationToken) { using var bitmap = request.Context.SoftwareBitmap; cancellationToken.ThrowIfCancellationRequested(); - if (this.disposedValue || request.Visibility.IsDisposed) + if (this.disposedValue || !this.OverlayVisible) { return; } @@ -384,7 +337,6 @@ private async ValueTask ProcessTranslationAsync( if (requests.Length > 0) { using var t = this.logger.LogDebugTime("翻訳"); - if (request.Visibility.IsDisposed) return; var translated = await this.translator.TranslateAsync(requests).ConfigureAwait(false); this.cache.AddRange(requests.Select(t => t.SourceText).Zip(translated)); var translatedIndex = 0; @@ -404,12 +356,11 @@ private async ValueTask ProcessTranslationAsync( catch (Exception e) { this.logger.LogError(e, "翻訳中にエラーが発生"); - if (await StopCaptureAsync(request.Visibility)) - { - this.trackingLoop.Dispose(); - this.translationLoop.Dispose(); - await ShowErrorAsync(Resources.FaildOverlay, e, string.Empty); - } + this.captureLoop.Dispose(); + this.trackingLoop.Dispose(); + await Application.Current.Dispatcher.InvokeAsync(this.capture.StopCapture); + this.translationLoop.Dispose(); + await ShowErrorAsync(Resources.FaildOverlay, e, string.Empty); return; } using (this.Filtering.EnterBusy()) @@ -423,7 +374,8 @@ private async ValueTask ProcessTranslationAsync( texts = await tmp.ToArrayAsync(cancellationToken); } cancellationToken.ThrowIfCancellationRequested(); - this.uiRequests.OnNext((texts, request.Visibility)); + if (request.Visibility.IsDisposed) return; + this.uiRequests.OnNext(texts); } private void UpdateOcrTexts(IEnumerable texts) @@ -458,7 +410,6 @@ protected virtual void Dispose(bool disposing) if (disposing) { // 実行中の処理から遅れて通知されても受け付けない。 - this.captureLoop.Dispose(); this.captureRequests.OnCompleted(); this.translationRequests.OnCompleted(); this.uiRequests.OnCompleted(); @@ -466,6 +417,7 @@ protected virtual void Dispose(bool disposing) { captureDisposable.Dispose(); } + this.captureLoop.Dispose(); this.trackingLoop.Dispose(); this.translationLoop.Dispose(); this.uiLoop.Dispose(); diff --git a/WindowTranslator/Modules/Main/OverlayMainWindow.xaml b/WindowTranslator/Modules/Main/OverlayMainWindow.xaml index 3bd46f83..278893f2 100644 --- a/WindowTranslator/Modules/Main/OverlayMainWindow.xaml +++ b/WindowTranslator/Modules/Main/OverlayMainWindow.xaml @@ -25,6 +25,7 @@ +