diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0fb856ff2..0e27f8d67 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,7 +26,7 @@ jobs: - name: Set up Ruby uses: ruby/setup-ruby@v1 with: - ruby-version: ruby-3.3.0 + ruby-version: "3.3" bundler-cache: true - name: Lint code for consistent style diff --git a/CHANGELOG.md b/CHANGELOG.md index 667be63ce..ba0fd5803 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **The RubyLLM provider works with ruby_llm 2.2** (`activeagent`). RubyLLM + 2.2.0 (2026-11-10) changed three things the provider leaned on. A + message's `tool_calls` became a `ToolCall::Collection` that yields the + calls rather than ID-and-call pairs, so every tool call read as `nil`; a + streamed delta now carries the provider's call ID in `provider_call_id` + and leaves `id` empty, so every fragment joined the latest call and + arguments doubled up; and a chat asks each tool for its `namespace`, + `qualified_name` and `deferred?`, which ActiveAgent's tool proxy did not + answer. The provider now iterates either shape, keys streamed fragments + by the provider's ID, and the proxy answers the three. ruby_llm 1.16 + through 2.1 behave as before. The evals conversation check also learned + 2.2's `acts_as_chat` modules, columns and reworded unavailable-tool error. - **`evals run` fails CI on a missed group gate** (`activeagent`). A run whose critical group slipped to 50% passed `evals run --min-pass-rate` whenever the other groups lifted the whole-run pass rate over the bar. diff --git a/lib/active_agent/providers/ruby_llm/tool_proxy.rb b/lib/active_agent/providers/ruby_llm/tool_proxy.rb index 9c6433a85..734fb837e 100644 --- a/lib/active_agent/providers/ruby_llm/tool_proxy.rb +++ b/lib/active_agent/providers/ruby_llm/tool_proxy.rb @@ -32,6 +32,21 @@ def provider_params alias_method :declared_parameters, :parameters alias_method :provider_options, :provider_params + # RubyLLM 2.2 registers a tool under its qualified name (namespace + # plus name) and asks each tool whether it defers. ActiveAgent's + # tools have no namespace and run when called. + def namespace + nil + end + + def qualified_name + name + end + + def deferred? + false + end + private def deep_stringify(obj) diff --git a/lib/active_agent/providers/ruby_llm_provider.rb b/lib/active_agent/providers/ruby_llm_provider.rb index 8aaecd7cb..e5e6908ac 100644 --- a/lib/active_agent/providers/ruby_llm_provider.rb +++ b/lib/active_agent/providers/ruby_llm_provider.rb @@ -152,15 +152,16 @@ def process_stream_chunk(chunk) # Handle tool calls in chunk if chunk.tool_calls&.any? message[:tool_calls] ||= [] - chunk.tool_calls.each do |key, tool_call| + each_tool_call(chunk.tool_calls) do |key, tool_call| # Only the first delta of a call has its ID and name; the rest # are fragments of its arguments. A fragment belongs to the call # that shares its stream key (the index RubyLLM 2 numbers calls # by), or, without one as RubyLLM 1.16 leaves it for OpenAI, to # the latest call. RubyLLM 1.16 cannot preserve interleaved # OpenAI calls: its parser drops their indices and fragments. - existing = if tool_call.id - message[:tool_calls].find { |tc| tc[:id] == tool_call.id } + call_id = streamed_tool_call_id(tool_call) + existing = if call_id + message[:tool_calls].find { |tc| tc[:id] == call_id } else @stream_tool_calls[key] || message[:tool_calls].last end @@ -169,7 +170,7 @@ def process_stream_chunk(chunk) existing[:function][:arguments] += fragment else existing = { - id: tool_call.id, + id: call_id, type: "function", function: { name: tool_call.name, @@ -492,7 +493,7 @@ def normalize_ruby_llm_response(response, model_id) # Handle tool calls if response.tool_calls&.any? - hash[:tool_calls] = response.tool_calls.map do |id, tc| + hash[:tool_calls] = each_tool_call(response.tool_calls).map do |id, tc| { id: id, type: "function", @@ -523,6 +524,41 @@ def normalize_ruby_llm_response(response, model_id) hash end + # Yields each of a message's tool calls with its key. RubyLLM 1.x and + # 2.0 to 2.1 keep them in a Hash keyed by call ID (or, on a streamed + # chunk, by stream index), so iterating yields pairs. RubyLLM 2.2's + # `ToolCall::Collection` yields the calls themselves and keeps the + # keys behind `each_key`, in the same order. + # + # @param calls [Hash, #each_key, nil] + # @return [Enumerator, Array] key and call pairs + def each_tool_call(calls, &block) + pairs = if calls.nil? + [] + elsif calls.is_a?(Hash) + calls.to_a + elsif calls.respond_to?(:each_key) + calls.each_key.to_a.zip(calls.each.to_a) + else + calls.each.to_a.map { |call| [ call.id, call ] } + end + block ? pairs.each(&block) : pairs + end + + # The provider's ID for a streamed tool call, present on its first + # delta and nil on the fragments. RubyLLM 2.2 keeps it in + # `provider_call_id` and leaves `id` (its own identifier, assigned when + # the message completes) nil on a chunk; earlier releases put the + # provider's ID in `id`. + # + # @param tool_call [RubyLLM::ToolCall] + # @return [String, nil] + def streamed_tool_call_id(tool_call) + provider_id = tool_call.provider_call_id if tool_call.respond_to?(:provider_call_id) + provider_id = nil if provider_id.to_s.empty? + provider_id || tool_call.id + end + # A streamed tool call's arguments as the text they add to the JSON # string the call builds up. Most APIs stream that string; Anthropic # opens a call with an empty Hash for its input, and Gemini sends the diff --git a/test/evals/ruby_llm_conversation_test.rb b/test/evals/ruby_llm_conversation_test.rb index 616cdf599..04ce707e7 100644 --- a/test/evals/ruby_llm_conversation_test.rb +++ b/test/evals/ruby_llm_conversation_test.rb @@ -23,7 +23,9 @@ class EvalsRubyLLMConversationTest < ActiveSupport::TestCase [ "lookup_order", false, nil ], [ "failing_hash", true, "backend timed out" ], [ "failing_json", true, "MCP tool failing_json timed out" ], - [ "github__search_issues", true, /\AModel tried to call unavailable tool `github__search_issues`/ ] + # RubyLLM 2.2 reworded its unavailable-tool error. + [ "github__search_issues", true, + /\A(Model tried to call unavailable tool|There is no tool named) `github__search_issues`/ ] ].freeze test "a conversation replays the same from RubyLLM messages and from acts_as_chat records" do diff --git a/test/evals/support/ruby_llm_conversation_script.rb b/test/evals/support/ruby_llm_conversation_script.rb index 4bc9bc7df..85f4362f0 100644 --- a/test/evals/support/ruby_llm_conversation_script.rb +++ b/test/evals/support/ruby_llm_conversation_script.rb @@ -113,13 +113,19 @@ def replay_json(messages) # RubyLLM's railtie requires these under Rails, in its order; a release # that renames one reads as a skip naming the file rather than a crash. # 2.1 added record (the base class its models inherit from), mcp_credential - # and provider_file; they are required where the release has them. + # and provider_file; 2.2 added queued_message, chat_file, file_store, + # chat_files, chat_column_guard, message_broadcasting and chunk_methods + # (acts_as_chat includes ChatFiles and prepends ChatColumnGuard). Each is + # required where the release has it. internals = if V2 - %w[record payload_helpers model tool_call usage batch mcp_credential provider_file chat_methods message_methods acts_as] + %w[record payload_helpers model tool_call usage batch mcp_credential provider_file queued_message + chat_file file_store chat_methods chat_files chat_column_guard message_methods message_broadcasting + chunk_methods acts_as] else %w[payload_helpers chat_methods message_methods model_methods tool_call_methods acts_as] end - since_2_1 = %w[record mcp_credential provider_file] + since_2_1 = %w[record mcp_credential provider_file queued_message chat_file file_store chat_files + chat_column_guard message_broadcasting chunk_methods] begin internals.each do |file| next if since_2_1.include?(file) && $LOAD_PATH.none? { |dir| File.exist?(File.join(dir, "ruby_llm/active_record/#{file}.rb")) } @@ -150,6 +156,9 @@ def replay_json(messages) t.json :pricing, default: {} t.json :metadata, default: {} t.timestamps + # RubyLLM 2.2 upserts the registry by this key (its install + # migration has always created the index). + t.index [ :provider, :model_id ], unique: true end create_table :chats do |t| t.references :ruby_llm_model, null: false @@ -168,6 +177,10 @@ def replay_json(messages) t.json :raw_content t.json :raw_reasoning t.string :finish_reason + # RubyLLM 2.2 columns (its install migration writes them). + t.string :cache_ttl + t.string :phase + t.text :continuation if LEGACY t.integer :tool_call_id t.integer :input_tokens @@ -179,11 +192,18 @@ def replay_json(messages) t.references :message, polymorphic: true, null: false, index: false t.references :result, polymorphic: true, index: false t.string :tool_call_id, null: false + # RubyLLM 2.2: the provider's ID beside RubyLLM's own, the tool's + # namespace, the claim and the MCP state. + t.string :provider_call_id + t.string :namespace t.string :name, null: false t.text :thought_signature t.string :approval t.boolean :remote, default: false, null: false + t.datetime :claimed_until t.json :arguments, default: {} + t.json :mcp_state + t.json :mcp_result t.timestamps end create_table :ruby_llm_usages do |t| @@ -229,6 +249,7 @@ class Message < ActiveRecord::Base t.json :pricing, default: {} t.json :metadata, default: {} t.timestamps + t.index [ :provider, :model_id ], unique: true end create_table :chats do |t| t.references :model @@ -252,6 +273,8 @@ class Message < ActiveRecord::Base end create_table :tool_calls do |t| t.string :tool_call_id, null: false + t.string :provider_call_id + t.string :namespace t.string :name, null: false t.text :thought_signature t.json :arguments, default: {}