Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -958,7 +958,7 @@ r = results[0]
r.judgment["choice"], r.judgment["confidence"], r.judgment["probabilities"]
```

For Jev on OpenRouter, use `DecisionTarget.openrouter("typesafe/jev-1.13")`, with the key in `OPENROUTER_API_KEY`. Provider routing, for example `{"provider": {"zdr": True}}`, goes in `extra_body`.
For Jev on OpenRouter, use `DecisionTarget.openrouter("typesafe/jev-1.13")`, with the key in `OPENROUTER_API_KEY`. Provider routing, for example `{"provider": {"zdr": True}}`, goes in `extra_body`. Each response's `raw` gains an `endpoint` block (scheme, host, port, and whether the standard proxy settings route it through a proxy), and a plain `http://` URL to a host other than loopback raises a warning.

`DecisionTarget`:
- checks the endpoint's limits before sending: 2–26 options per question, and 64 KiB per request for Ollama. A scenario over a limit is recorded as an error; its documents are never shortened.
Expand Down Expand Up @@ -992,6 +992,8 @@ Use `max_turns=1` with `choice_match`. Follow-up turns need an `auditor_model` o
local servers can use the same target. No timestamps are required. Provider/model
support for languages, formats and optional parameters varies; unsupported
requests are reported as errors rather than silently changing the input.
The file is uploaded as `audio` plus its lowercased extension, so the local
filename is not sent with the upload.

```python
from simpleaudit import Auditor, ASRTarget
Expand Down
2 changes: 2 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,8 @@ target-version = ["py311", "py312", "py313"]
testpaths = ["tests"]
addopts = "--cov=simpleaudit --cov-report=html --cov-report=term-missing"
asyncio_mode = "auto"
# Tests use plain-http stub hosts; the targets' cleartext warning is expected there.
filterwarnings = ['ignore:target endpoint http\S+ uses plain http:UserWarning']

[tool.coverage.run]
source = ["simpleaudit"]
Expand Down
77 changes: 77 additions & 0 deletions simpleaudit/targets/_http.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
"""HTTP plumbing shared by the targets: client creation and endpoint provenance.

``describe_endpoint`` records where a request went (scheme, host, port, and
whether the standard proxy settings route it through a proxy) so an audit
record can say which endpoint answered. It never includes the path, query,
userinfo, headers or keys.
"""

from __future__ import annotations

import ipaddress
import urllib.request
import warnings
from typing import Any
from urllib.parse import urlsplit

_DEFAULT_PORTS = {"http": 80, "https": 443}


def make_client(timeout: float) -> Any:
"""A fresh ``httpx.AsyncClient(timeout=timeout)``, as the targets always created."""
import httpx

return httpx.AsyncClient(timeout=timeout)


def _is_loopback(host: str) -> bool:
if host.lower() == "localhost":
return True
try:
return ipaddress.ip_address(host).is_loopback
except ValueError:
return False


def describe_endpoint(url: str) -> dict[str, Any]:
"""``{"scheme", "host", "port", "via_proxy"}`` for ``url``.

``via_proxy`` follows ``urllib.request.getproxies()`` / ``proxy_bypass()``
(the environment's proxy variables); an injected client's own proxy
configuration is not visible here.
"""
parts = urlsplit(url)
scheme = parts.scheme.lower()
host = parts.hostname or ""
proxies = urllib.request.getproxies()
proxied = bool(proxies.get(scheme) or proxies.get("all"))
return {
"scheme": scheme,
"host": host,
"port": parts.port or _DEFAULT_PORTS.get(scheme),
"via_proxy": proxied and not urllib.request.proxy_bypass(host),
}


def warn_if_cleartext(url: str) -> None:
"""Warn when ``url`` is plain http to a host other than loopback."""
parts = urlsplit(url)
if parts.scheme.lower() == "http" and not _is_loopback(parts.hostname or ""):
warnings.warn(
f"target endpoint http://{parts.hostname} uses plain http: requests "
"and responses to it are not encrypted",
UserWarning,
stacklevel=3,
)


def with_endpoint(raw: Any, url: str) -> dict[str, Any]:
"""``raw`` with an ``endpoint`` block added; provider fields are never overwritten.

A dict without an ``endpoint`` key gets one; anything else is wrapped as
``{"response": raw, "endpoint": ...}``.
"""
endpoint = describe_endpoint(url)
if isinstance(raw, dict) and "endpoint" not in raw:
return {**raw, "endpoint": endpoint}
return {"response": raw, "endpoint": endpoint}
13 changes: 6 additions & 7 deletions simpleaudit/targets/asr.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
from typing import Any
from urllib.parse import unquote, urlparse

from ._http import make_client, warn_if_cleartext, with_endpoint
from .base import TargetContext, TargetResponse


Expand Down Expand Up @@ -35,6 +36,7 @@ def __init__(
) -> None:
self.model = model
self.url = base_url.rstrip("/") + "/audio/transcriptions"
warn_if_cleartext(self.url)
self.api_key = api_key
self.timeout = timeout
self.options = dict(options or {})
Expand Down Expand Up @@ -63,8 +65,6 @@ async def send(
documents: list[str | dict[str, Any]] | None = None,
response_format: dict[str, Any] | None = None,
) -> TargetResponse:
import httpx

if file_uri is None:
file_uri = next(
(
Expand Down Expand Up @@ -102,9 +102,7 @@ async def send(
headers = {"Authorization": f"Bearer {self.api_key}"} if self.api_key else {}
if context:
headers.update(context.trace_headers)
client = (
self._client if self._client is not None else httpx.AsyncClient(timeout=self.timeout)
)
client = self._client if self._client is not None else make_client(self.timeout)
try:
with path.open("rb") as audio:
response = await client.post(
Expand All @@ -113,7 +111,8 @@ async def send(
headers=headers,
files={
"file": (
path.name,
# Neutral name: the local filename stays local.
"audio" + path.suffix.lower(),
audio,
mimetypes.guess_type(path.name)[0] or "application/octet-stream",
)
Expand All @@ -124,7 +123,7 @@ async def send(
payload = response.json()
if not isinstance(payload, dict) or not isinstance(payload.get("text"), str):
raise ValueError("Transcription response must contain a string 'text' field")
return TargetResponse(content=payload["text"], raw=payload)
return TargetResponse(content=payload["text"], raw=with_endpoint(payload, self.url))
finally:
if self._client is None:
await client.aclose()
8 changes: 4 additions & 4 deletions simpleaudit/targets/decision.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
from typing import Any

from ..context_marks import parse_documents
from ._http import make_client, warn_if_cleartext, with_endpoint
from .base import TargetContext, TargetResponse

#: Ollama's request-body cap for System One requests without images.
Expand Down Expand Up @@ -70,6 +71,7 @@ def __init__(
max_body_bytes: int | None = None,
) -> None:
self.url = url
warn_if_cleartext(url)
self.model = model
self.headers = dict(headers or {})
if api_key:
Expand Down Expand Up @@ -177,9 +179,7 @@ async def send(
if client := self._client:
pass
else:
import httpx

client = httpx.AsyncClient(timeout=self.timeout)
client = make_client(self.timeout)
try:
resp = await client.post(self.url, content=payload, headers=headers)
if resp.status_code >= 400:
Expand All @@ -204,7 +204,7 @@ async def send(
usage = data.get("usage") or {}
return TargetResponse(
content=f"{choice}: {description}" if description else choice,
raw=data,
raw=with_endpoint(data, self.url),
input_tokens=usage.get("input_tokens"),
output_tokens=usage.get("output_tokens"),
decision=answer,
Expand Down
8 changes: 4 additions & 4 deletions simpleaudit/targets/http.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import json
from typing import Any, Callable, Dict, List, Optional, Union

from ._http import make_client, warn_if_cleartext, with_endpoint
from .base import Target, TargetContext, TargetResponse


Expand Down Expand Up @@ -67,6 +68,7 @@ def __init__(
token_paths: Optional[tuple] = None,
) -> None:
self.url = url
warn_if_cleartext(url)
self.method = method.upper()
self.headers = dict(headers or {})
self.request_template = dict(request_template or {})
Expand Down Expand Up @@ -145,9 +147,7 @@ async def send(
if client := self._client:
pass
else:
import httpx

client = httpx.AsyncClient(timeout=self.timeout)
client = make_client(self.timeout)
try:
resp = await client.request(self.method, self.url, json=body, headers=headers)
resp.raise_for_status()
Expand All @@ -172,7 +172,7 @@ async def send(

return TargetResponse(
content=content,
raw=data,
raw=with_endpoint(data, self.url),
input_tokens=int(in_tok) if in_tok is not None else None,
output_tokens=int(out_tok) if out_tok is not None else None,
)
97 changes: 97 additions & 0 deletions tests/test_endpoint_provenance.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
"""Endpoint provenance in target responses, the cleartext warning, neutral ASR filenames."""

import warnings

import httpx
import pytest

from simpleaudit import ASRTarget, DecisionTarget, HTTPAppTarget
from simpleaudit.targets._http import describe_endpoint

from .test_decision_target import _context, _endpoint, _send

URL = "https://user:secret@api.example.com:8443/v1/chat?key=abc123"
PROXY_VARS = ("HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "NO_PROXY")


@pytest.fixture
def proxy_env(monkeypatch):
for name in PROXY_VARS:
monkeypatch.delenv(name, raising=False)
monkeypatch.delenv(name.lower(), raising=False)
# An environment proxy for another scheme keeps the system settings out of it.
monkeypatch.setenv("HTTP_PROXY", "http://proxy.test:3128")
return monkeypatch


@pytest.mark.parametrize(
"https_proxy,no_proxy,expected",
[(None, None, False), ("http://proxy.test:3128", None, True),
("http://proxy.test:3128", "api.example.com", False)],
)
def test_describe_endpoint_and_via_proxy(proxy_env, https_proxy, no_proxy, expected):
if https_proxy:
proxy_env.setenv("HTTPS_PROXY", https_proxy)
if no_proxy:
proxy_env.setenv("NO_PROXY", no_proxy)
assert describe_endpoint(URL) == {
"scheme": "https", "host": "api.example.com", "port": 8443, "via_proxy": expected
}


def _http_target(payload):
client = httpx.AsyncClient(transport=httpx.MockTransport(lambda r: httpx.Response(200, **payload)))
return HTTPAppTarget(URL, response_path="answer", client=client)


@pytest.mark.asyncio
async def test_endpoint_block_is_merged_and_carries_no_path_query_or_key(proxy_env):
raw = (await _http_target({"json": {"answer": "ok", "id": 7}}).send(user="q")).raw
assert raw["answer"] == "ok" and raw["id"] == 7
assert raw["endpoint"]["host"] == "api.example.com"
assert not any(s in repr(raw["endpoint"]) for s in ("secret", "user", "/v1", "key", "abc123"))


@pytest.mark.asyncio
async def test_non_dict_or_conflicting_raw_is_wrapped_not_overwritten(proxy_env):
raw = (await _http_target({"text": "plain"}).send(user="q")).raw
assert raw["response"] == "plain" and raw["endpoint"]["scheme"] == "https"
raw = (await _http_target({"json": {"answer": "ok", "endpoint": "provider"}}).send(user="q")).raw
assert raw["response"]["endpoint"] == "provider" and raw["endpoint"]["port"] == 8443


def test_decision_response_carries_the_endpoint(proxy_env):
client, _ = _endpoint()
raw = _send(DecisionTarget(URL, "clef", client=client), context=_context()).raw
assert raw["answers"]["verdict"]["choice"] == "yes" and raw["endpoint"]["port"] == 8443


def test_plain_http_to_a_remote_host_warns():
with pytest.warns(UserWarning, match="http://example.com uses plain http"):
HTTPAppTarget("http://example.com/chat")


@pytest.mark.parametrize(
"url", ["http://localhost:11434", "http://127.0.0.5/v1", "http://[::1]:8000", "https://example.com"]
)
def test_loopback_and_https_do_not_warn(url):
with warnings.catch_warnings():
warnings.simplefilter("error")
DecisionTarget(url, "clef")
ASRTarget("m", base_url=url)


@pytest.mark.asyncio
async def test_asr_upload_filename_is_neutral(tmp_path):
audio = tmp_path / "Interview Kari Nordmann.WAV"
audio.write_bytes(b"RIFF-audio")
seen = []

def handler(request):
seen.append(request.content)
return httpx.Response(200, json={"text": "ok"})

async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client:
await ASRTarget("m", client=client).send(user="", file_uri=str(audio))
assert b'filename="audio.wav"' in seen[0]
assert b"Kari" not in seen[0] and b"Interview" not in seen[0]
Loading