diff --git a/test/irb/test_irb.rb b/test/irb/test_irb.rb
index 575fda11b..28fa72d96 100644
--- a/test/irb/test_irb.rb
+++ b/test/irb/test_irb.rb
@@ -994,11 +994,18 @@ class ContextModeTest < TestIRB::IntegrationTestCase
     def test_context_mode_ruby_box
       omit if RUBY_VERSION < "4.0.0"
       @envs['RUBY_BOX'] = '1'
-      # Ruby::Box now initializes RubyGems per box upstream. Avoid loading bundler context into test execution context.
-      %w[BUNDLER_SETUP BUNDLER_VERSION BUNDLE_BIN_PATH BUNDLE_GEMFILE RUBYOPT].each do |env|
-        @envs[env] = nil
+      # bundler/setup loaded via RUBYOPT fails inside Ruby::Box (Ruby::Box
+      # initializes RubyGems per box, and Gem::Specification is not defined when
+      # Bundler evaluates irb.gemspec), so remove the bundler context from the
+      # child process. Keep the env intact when not under bundler: in ruby/ruby's
+      # test-bundled-gems, inherited RUBYOPT/RUBYLIB carry the build-tree load
+      # paths without which the child cannot require prism.
+      if ENV['RUBYOPT']&.include?('bundler/setup')
+        %w[BUNDLER_SETUP BUNDLER_VERSION BUNDLE_BIN_PATH BUNDLE_GEMFILE RUBYOPT].each do |env|
+          @envs[env] = nil
+        end
+        @envs['RUBYLIB'] = Gem.loaded_specs.fetch('reline').full_require_paths.join(File::PATH_SEPARATOR)
       end
-      @envs['RUBYLIB'] = Gem.loaded_specs.fetch('reline').full_require_paths.join(File::PATH_SEPARATOR)
 
       write_rc <<~'RUBY'
         IRB.conf[:CONTEXT_MODE] = 5