feat(streaming_hints): add streaming hint injection plugin

.github/workflows/test.yml

@@ -53,6 +53,23 @@ jobs:
         python tests/test_ci_quick.py
         python -m pytest tests/test_plugins.py -v --tb=short || python tests/test_plugins.py
         python -m pytest tests/test_proxy_plugin.py -v --tb=short
+        python -m pytest tests/test_streaming_hints.py -v --tb=short
+        python -m pytest tests/test_plugin_gating.py -v --tb=short
+        python -m pytest tests/test_max_completion_tokens.py -v --tb=short
+        python -m pytest tests/test_bon_candidate_filtering.py -v --tb=short
+        python -m pytest tests/test_compact_plugin.py -v --tb=short
+        python -m pytest tests/test_completions_endpoint.py -v --tb=short
+        python -m pytest tests/test_entropy_decoding_sampling.py -v --tb=short
+        python -m pytest tests/test_mars_agent_routing.py -v --tb=short
+        python -m pytest tests/test_mars_aggregation_sampling.py -v --tb=short
+        python -m pytest tests/test_mars_imo25.py -v --tb=short
+        python -m pytest tests/test_mars_workspace_verification.py -v --tb=short
+        python -m pytest tests/test_mcts_selection.py -v --tb=short
+        python -m pytest tests/test_privacy_plugin_performance.py -v --tb=short
+        python -m pytest tests/test_readurls_ssrf.py -v --tb=short
+        python -m pytest tests/test_rstar_backpropagation.py -v --tb=short
+        python -m pytest tests/test_rstar_numeric_answers.py -v --tb=short
+        python -m pytest tests/test_z3_solver_guard.py -v --tb=short
         python tests/test_approaches.py
         python tests/test_reasoning_simple.py
         python tests/test_batching.py
@@ -125,7 +142,7 @@ jobs:
         OPTILLM_API_KEY=optillm python tests/test_reasoning_integration.py
         OPTILLM_API_KEY=optillm python tests/test_json_plugin.py
         OPTILLM_API_KEY=optillm python tests/test_n_parameter.py
-        OPTILLM_API_KEY=optillm python -m pytest tests/test_api_compatibility.py -v --tb=short || echo "API compatibility tests require pytest"
+        OPTILLM_API_KEY=optillm python -m pytest tests/test_api_compatibility.py -v --tb=short
         OPTILLM_API_KEY=optillm python tests/test.py --approaches none --single-test "Simple Math Problem" || echo "Main test completed"
 
         # Run SSL config tests (no server needed but requires proper env setup)
@@ -256,7 +273,6 @@ jobs:
   mcp-tests:
     runs-on: ubuntu-latest
     needs: unit-tests
-    if: github.event_name == 'push' && github.ref == 'refs/heads/main'  # Only run on main branch pushes (secrets available)
     strategy:
       matrix:
         python-version: ['3.12']
@@ -296,5 +312,6 @@ jobs:
         echo "MCP tests completed successfully!"
       env:
         OPTILLM_API_KEY: optillm
+        # Not available on fork PRs, where the one test that needs it skips itself
         GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
         HF_TOKEN: ${{ secrets.HF_TOKEN }}

docs/llm-gateway.md

@@ -0,0 +1,74 @@
+# Using OptiLLM as an LLM gateway
+
+OptiLLM's proxy mode gives you one self-hosted, OpenAI-compatible endpoint in front of any number of providers, with load balancing, failover, and health checks. It is free and Apache-2.0 licensed, and you pay your providers directly.
+
+OptiLLM combines routing and inference optimization in the same service. The same request that is routed across providers can also run through [20+ state-of-the-art reasoning techniques](../README.md#techniques), so your gateway can improve answers as well as deliver them.
+
+## Quick start
+
+Create `~/.optillm/proxy_config.yaml`:
+
+```yaml
+providers:
+  - name: openai
+    base_url: https://api.openai.com/v1
+    api_key: ${OPENAI_API_KEY}
+    weight: 3
+
+  - name: together
+    base_url: https://api.together.xyz/v1
+    api_key: ${TOGETHER_API_KEY}
+    weight: 1
+    model_map:
+      gpt-4o-mini: meta-llama/Llama-3.3-70B-Instruct-Turbo
+
+  - name: local
+    base_url: http://localhost:8080/v1
+    api_key: none
+    max_concurrent: 2
+
+routing:
+  strategy: weighted  # or round_robin, failover
+  health_check:
+    enabled: true
+    interval: 30
+```
+
+Start the server with the proxy as the default approach and protect it with an API key:
+
+```bash
+optillm --approach proxy --optillm-api-key "$GATEWAY_KEY"
+```
+
+Point any OpenAI client at it:
+
+```python
+from openai import OpenAI
+
+client = OpenAI(api_key=GATEWAY_KEY, base_url="http://localhost:8000/v1")
+
+response = client.chat.completions.create(
+    model="gpt-4o-mini",
+    messages=[{"role": "user", "content": "Hello"}],
+)
+```
+
+If a provider fails, the request moves to the next one, and then to the server's default client (`--base-url`) if one is available.
+
+## Adding inference optimization
+
+Start the server without `--approach proxy` and select the proxy per request, on its own or wrapped around a technique:
+
+```python
+# Route only
+client.chat.completions.create(model="proxy-gpt-4o-mini", messages=messages)
+
+# Run Mixture of Agents, sending its calls through the proxy
+client.chat.completions.create(
+    model="gpt-4o-mini",
+    messages=messages,
+    extra_body={"optillm_approach": "proxy", "proxy_wrap": "moa"},
+)
+```
+
+See the [proxy plugin documentation](../optillm/plugins/proxy/README.md) for the full configuration reference.

optillm/__init__.py

@@ -1,5 +1,5 @@
 # Version information
-__version__ = "0.4.0"
+__version__ = "0.4.1"
 
 import os as _os
 

optillm/inference.py

@@ -1872,6 +1872,10 @@ def create(
             ) -> ChatCompletion:
                 """Create a chat completion with OpenAI-compatible parameters"""
                 logger.info("Starting chat completion creation")
+                # max_completion_tokens is the preferred name for the same limit
+                max_completion_tokens = kwargs.pop("max_completion_tokens", None)
+                if max_completion_tokens is not None:
+                    max_tokens = max_completion_tokens
                 if stream:
                     raise NotImplementedError("Streaming is not yet supported")
 

optillm/plugins/mcp_plugin.py

@@ -192,7 +192,6 @@ def load_config(self) -> bool:
 
             with open(self.config_path, 'r') as f:
                 config_data = f.read()
-                logger.debug(f"Raw config data: {config_data}")
                 config = json.loads(config_data)
 
             # Set log level
@@ -204,7 +203,7 @@ def load_config(self) -> bool:
             servers_config = config.get("mcpServers", {})
             for server_name, server_config in servers_config.items():
                 self.servers[server_name] = ServerConfig.from_dict(server_config)
-                logger.debug(f"Loaded server config for {server_name}: {server_config}")
+                logger.debug(f"Loaded server config for {server_name} (transport: {self.servers[server_name].transport})")
 
             logger.info(f"Loaded configuration with {len(self.servers)} servers")
             return True
@@ -398,7 +397,7 @@ async def connect_sse(self) -> bool:
         """Connect to server using SSE transport and discover capabilities"""
         logger.info(f"Connecting to SSE server: {self.server_name}")
         logger.debug(f"SSE URL: {self.config.url}")
-        logger.debug(f"Headers: {self.config.headers}")
+        logger.debug(f"Headers: {list(self.config.headers or {})}")
 
         if not self.config.url:
             logger.error(f"SSE transport requires URL for server {self.server_name}")
@@ -435,7 +434,6 @@ async def connect_websocket(self) -> bool:
 
     async def connect_stdio_native(self) -> bool:
         """Connect using stdio transport with local executable"""
-        logger.debug(f"Server configuration: {vars(self.config)}")
 
         # Validate stdio configuration
         if not self.config.command:
@@ -455,7 +453,7 @@ async def connect_stdio_native(self) -> bool:
 
         logger.debug(f"Using command: {full_command}")
         logger.debug(f"Arguments: {self.config.args}")
-        logger.debug(f"Environment: {self.config.env}")
+        logger.debug(f"Environment variables: {list(self.config.env or {})}")
 
         # Create server parameters
         server_params = StdioServerParameters(

optillm/plugins/proxy_plugin.py

@@ -37,6 +37,18 @@ def _request_messages(system_prompt: str, initial_query: str, messages: Optional
         {"role": "user", "content": initial_query}
     ]
 
+def _passthrough_config(request_config: Optional[dict]) -> dict:
+    """
+    Build the kwargs forwarded to a provider. Streaming is stripped to force a
+    complete response (server.py converts it to SSE if needed), and the max_tokens
+    copy of max_completion_tokens is dropped because o-series and gpt-5 models reject it.
+    """
+    api_config = dict(request_config or {})
+    api_config.pop('stream', None)
+    if api_config.get('max_completion_tokens') is not None:
+        api_config.pop('max_tokens', None)
+    return api_config
+
 def run(system_prompt: str, initial_query: str, client, model: str,
         request_config: dict = None, messages: list = None) -> Tuple[str, int]:
     """
@@ -65,9 +77,7 @@ def run(system_prompt: str, initial_query: str, client, model: str,
 
         if not config.get('providers'):
             logger.warning(f"No providers configured in {ProxyConfig._config_path}, falling back to original client")
-            # Strip stream parameter to force complete response
-            api_config = dict(request_config or {})
-            api_config.pop('stream', None)
+            api_config = _passthrough_config(request_config)
 
             response = client.chat.completions.create(
                 model=model,
@@ -143,10 +153,7 @@ def run(system_prompt: str, initial_query: str, client, model: str,
         # message support and merges the system prompt when needed.
         logger.info(f"Direct proxy routing for model: {model}")
 
-        # Strip stream parameter to force complete response
-        # server.py will handle converting to SSE streaming format if needed
-        api_config = dict(request_config or {})
-        api_config.pop('stream', None)
+        api_config = _passthrough_config(request_config)
 
         response = proxy_client.chat.completions.create(
             model=model,
@@ -162,9 +169,7 @@ def run(system_prompt: str, initial_query: str, client, model: str,
         logger.error(f"Proxy plugin error: {e}", exc_info=True)
         # Fallback to original client
         logger.info("Falling back to original client")
-        # Strip stream parameter to force complete response
-        api_config = dict(request_config or {})
-        api_config.pop('stream', None)
+        api_config = _passthrough_config(request_config)
 
         response = client.chat.completions.create(
             model=model,

optillm/plugins/streaming_hints/README.md

@@ -0,0 +1,73 @@
+# Streaming Hints
+
+Injects hints from a memory bank into the model's reasoning while it is being generated, inspired by [this idea from Will Brown](https://x.com/willccbb/status/1940557166248972387). Instead of putting every piece of guidance in the prompt up front, the plugin watches the reasoning and adds one targeted hint at the moment the model hesitates or moves to a new step.
+
+This is the inference half of the approach. The retriever is rule-based for now; training the injection policy with RL is future work (see below).
+
+## How it works
+
+1. The model answers the query in segments of at most `segment_tokens` tokens.
+2. Each segment is fed sentence by sentence to the `StreamingHintRetriever`.
+3. A sentence is an **injection point** when at least `min_gap_chars` characters have passed since the last hint and the sentence contains an uncertainty signal ("wait", "hmm", "not sure", "let me check", ...) or ends a paragraph.
+4. At an injection point the retriever queries the `HintMemoryBank` with the problem plus the most recent reasoning. If the best unused hint scores at least `hint_threshold`, the segment is cut after that sentence and the hint is appended as `<hint>...</hint>`.
+5. Generation continues from the cut, with the reasoning so far (hints included) in the prompt. This repeats until the model finishes, `max_segments` is reached, or `max_tokens` is spent.
+
+Segments are used instead of a token stream so the plugin works with every client optillm supports, including the local inference server, which does not stream. The cost is that up to one segment of text after an injection point is generated and then discarded.
+
+The memory bank ranks hints by TF-IDF cosine similarity over the hint text and its keywords. It is pure Python, so the plugin has no extra dependencies.
+
+## Usage
+
+```python
+from openai import OpenAI
+
+client = OpenAI(api_key="sk-...", base_url="http://localhost:8000/v1")
+
+response = client.chat.completions.create(
+    model="streaming_hints-gpt-4o-mini",
+    messages=[{"role": "user", "content": "A train travels 120 km in 1.5 hours. What is its speed in m/s?"}],
+    extra_body={"hint_threshold": 0.25, "show_hints": True},
+)
+```
+
+It combines with other approaches like any plugin, for example `streaming_hints&cot_reflection-gpt-4o-mini`.
+
+## Configuration
+
+All options are read from the request body (`extra_body`).
+
+| Option | Default | Description |
+|---|---|---|
+| `hint_threshold` | `0.2` | Minimum similarity score for a hint to be injected |
+| `max_hints` | `3` | Maximum number of hints injected per request |
+| `min_gap_chars` | `200` | Minimum characters of reasoning between two injections |
+| `segment_tokens` | `512` | Maximum tokens generated per segment |
+| `max_segments` | `8` | Maximum number of segments per request |
+| `hints_path` | bundled `hints.json` | Path to a custom hint bank |
+| `show_hints` | `false` | Return the reasoning with the `<hint>` tags instead of the model text only |
+
+`max_tokens`, if set, caps the completion tokens across all segments.
+
+## Custom hints
+
+A hint bank is a JSON list. `keywords` count twice as much as words in `text` when matching.
+
+```json
+[
+  {
+    "id": "math-units",
+    "domain": "math",
+    "text": "Write down the units of every quantity and check both sides match.",
+    "keywords": ["units", "speed", "rate", "convert"]
+  }
+]
+```
+
+The bundled bank has 20 general hints for math, coding and logical reasoning. Domain-specific banks will usually do better.
+
+## Future work
+
+- **RL-trained retriever.** Learn when to inject and which hint to pick from downstream task accuracy, with and without hints as the reward signal. `Hint.uses` is tracked as a starting point for usage statistics.
+- **Embedding retrieval.** A bank backed by sentence embeddings can replace TF-IDF behind the same `search` interface.
+- **Token streaming.** For providers that stream, the retriever can be fed token deltas directly and the stream cancelled at the injection point, which avoids the discarded tail of a segment.
+- **Evaluation.** Benchmark results (GSM8K, MATH, HumanEval, MMLU) against the plain model and against putting all hints in the prompt.

optillm/plugins/streaming_hints/__init__.py

@@ -0,0 +1,12 @@
+"""
+Streaming Hints Plugin for OptiLLM
+
+Monitors the model's reasoning as it is generated and injects relevant hints
+from a memory bank at moments of uncertainty or between reasoning steps.
+"""
+
+from .memory_bank import Hint, HintMemoryBank
+from .retriever import Injection, StreamingHintRetriever
+from .streaming_hints import run_streaming_hints
+
+__all__ = ['Hint', 'HintMemoryBank', 'Injection', 'StreamingHintRetriever', 'run_streaming_hints']