fix: report lease loss in all turn outcomes

Catch lost fences around the full executor path, including failure and
rejection persistence. Preserve recovery and report each lost attempt so
instrumentation can stop a repeatedly interrupted worker.

Before the fix, two new recovery tests missed activation.lost and two
callback shutdown cases required their outer time limit. All four pass
now. Full rake passes: 940 tests, 3940 assertions, 28 conditional skips.

Author: cardmagic

CHANGELOG.md

@@ -11,7 +11,8 @@
   can proceed. Lease loss alone still does not dead-letter a message; handler
   failure attempt limits, setting names, defaults, and schema stay unchanged.
 - Emit `activation.lost` with the message ID and total attempt count when a
-  turn loses its fence, matching JavaScript instrumentation. Shutdown requested
+  turn loses its fence, including while saving a handler failure or rejection,
+  matching JavaScript instrumentation. Shutdown requested
   by that callback stops the worker before another pass. Skip a cached actor
   whose first message is delayed so later ready messages cannot starve peers.
 

docs/roadmap.md

@@ -28,7 +28,8 @@
   fenced commits. Interrupted first attempts recover immediately; repeated
   interruptions use durable retry delays from total claim attempts minus one,
   preserving mailbox order and progress for other actors. Lease loss emits
-  `activation.lost` with the message ID and attempt and has no hard retry cap;
+  `activation.lost` with the message ID and attempt, including losses while
+  saving a handler failure or rejection. There is no hard retry cap;
   applications needing one must impose a stopping policy.
 - Bounded activation passes, idle cache, hot-actor yield, and process records
 - At-least-once retries, terminal domain rejection, strict poison ordering,

lib/solid_objects/executor.rb

@@ -16,6 +16,18 @@ def initialize(activation:, message:)
 
     # @rbs () -> bool
     def call
+      execute
+    rescue LostActivation
+      SolidObjects.instrument(:"activation.lost", **instrumentation_payload, duration_milliseconds: elapsed_milliseconds)
+      raise
+    end
+
+    private
+
+    attr_reader :activation, :message
+
+    # @rbs () -> bool
+    def execute
       state_before = actor.state.to_h
       observables_before = actor.observable_values
       message_context = MessageContext.new(
@@ -39,10 +51,7 @@ def call
         state_changed: state_after.value != state_before
       )
       true
-    rescue CommittedTransactionError
-      raise
-    rescue LostActivation
-      SolidObjects.instrument(:"activation.lost", **instrumentation_payload, duration_milliseconds: elapsed_milliseconds)
+    rescue CommittedTransactionError, LostActivation
       raise
     rescue Rejected => rejection
       activation.restore_state(state_before) if state_before
@@ -56,10 +65,6 @@ def call
       false
     end
 
-    private
-
-    attr_reader :activation, :message
-
     # @rbs () -> Actor
     def actor
       activation.actor

test/integration/activation_recovery_backoff_test.rb

@@ -117,37 +117,74 @@ def increment(label:)
     end
   end
 
-  test "emits each lost message attempt and stops from its instrumentation callback" do
-    worker = new_worker
-    reference = Counter.ref("one").async.increment(label: "slow")
-    observed = []
-    SolidObjects.configuration.retry_delay = ->(_) { 0 }
-    SolidObjects.configuration.instrumentation = ->(event) do
-      next unless event.fetch("name") == "solid_objects.activation.lost"
+  %w[completion failure rejection].each do |outcome|
+    test "reports each lost #{outcome} and stops from its instrumentation callback" do
+      worker = new_worker
+      reference = Counter.ref("one").async.increment(label: "slow")
+      Counter.before_commit = ->(_) do
+        @clock[:now] += 0.201
+        raise SolidObjects::Rejected.new(code: "declined", message: "declined") if outcome == "rejection"
+        raise "handler failed" if outcome == "failure"
+      end
+      observed = []
+      SolidObjects.configuration.retry_delay = ->(_) { 0 }
+      SolidObjects.configuration.instrumentation = ->(event) do
+        next unless event.fetch("name") == "solid_objects.activation.lost"
 
-      observed << event
-      worker.request_shutdown if observed.length == 3
-    end
-    forced_shutdown = false
-    guard = Thread.new do
-      sleep 2
-      forced_shutdown = true
-      worker.request_shutdown
+        observed << event
+        worker.request_shutdown if observed.length == 3
+      end
+      forced_shutdown = false
+      guard = Thread.new do
+        sleep 2
+        forced_shutdown = true
+        worker.request_shutdown
+      end
+      begin
+        worker.run
+      ensure
+        guard.kill
+        guard.join
+      end
+      assert_not forced_shutdown
+      assert_predicate worker, :stopped?
+      assert_equal [ 1, 2, 3 ], observed.map { |event| event.fetch("attempt") }
+      assert observed.all? { |event| event.fetch("messageId") == reference.id.to_s }
+      observed.each { |event| assert_portable_attributes(event) }
+      assert worker.shutdown_requested?
+      assert_empty stored_message(reference).instance.state
+      assert_empty SolidObjects::DeadLetter.all
     end
-    begin
-      worker.run
-    ensure
-      guard.kill
-      guard.join
+  end
+
+  %w[failure rejection].each do |outcome|
+    test "reports lease loss while saving a handler #{outcome} and recovers" do
+      worker = new_worker
+      reference = Counter.ref("one").async.increment(label: "slow")
+      Counter.before_commit = ->(_) do
+        @clock[:now] += 0.201
+        raise SolidObjects::Rejected.new(code: "declined", message: "declined") if outcome == "rejection"
+
+        raise "handler failed"
+      end
+      assert_equal 0, worker.run_once
+      losses = @events.select { |event| event.fetch("name") == "solid_objects.activation.lost" }
+      assert_equal [ 1 ], losses.map { |event| event.fetch("attempt") }
+      assert_equal [ reference.id.to_s ], losses.map { |event| event.fetch("messageId") }
+      losses.each { |event| assert_portable_attributes(event) }
+      assert_empty stored_message(reference).instance.state
+      assert_nil stored_message(reference).error
+      assert_nil stored_message(reference).rejection
+      assert_empty SolidObjects::DeadLetter.all
+      assert_empty SolidObjects::Effect.all
+      assert_not @events.any? { |event| %w[solid_objects.message.failed solid_objects.message.rejected].include?(event.fetch("name")) }
+
+      Counter.before_commit = nil
+      assert_equal 1, worker.run_once
+      assert_equal 1, stored_message(reference).result
+      assert_equal 2, stored_message(reference).attempt_count
+      assert_equal 1, SolidObjects::Effect.count
     end
-    assert_not forced_shutdown
-    assert_predicate worker, :stopped?
-    assert_equal [ 1, 2, 3 ], observed.map { |event| event.fetch("attempt") }
-    assert observed.all? { |event| event.fetch("messageId") == reference.id.to_s }
-    observed.each { |event| assert_portable_attributes(event) }
-    assert worker.shutdown_requested?
-    assert_empty stored_message(reference).instance.state
-    assert_empty SolidObjects::DeadLetter.all
   end
 
   test "keeps a handler failure terminal with one allowed attempt" do