diff --git a/README.md b/README.md index 05ebf74..1dcb4df 100644 --- a/README.md +++ b/README.md @@ -958,7 +958,7 @@ r = results[0] r.judgment["choice"], r.judgment["confidence"], r.judgment["probabilities"] ``` -For Jev on OpenRouter, use `DecisionTarget.openrouter("typesafe/jev-1.13")`, with the key in `OPENROUTER_API_KEY`. Provider routing, for example `{"provider": {"zdr": True}}`, goes in `extra_body`. +For Jev on OpenRouter, use `DecisionTarget.openrouter("typesafe/jev-1.13")`, with the key in `OPENROUTER_API_KEY`. Provider routing, for example `{"provider": {"zdr": True}}`, goes in `extra_body`. Each response's `raw` gains an `endpoint` block (scheme, host, port, and whether the standard proxy settings route it through a proxy), and a plain `http://` URL to a host other than loopback raises a warning. `DecisionTarget`: - checks the endpoint's limits before sending: 2–26 options per question, and 64 KiB per request for Ollama. A scenario over a limit is recorded as an error; its documents are never shortened. @@ -992,6 +992,8 @@ Use `max_turns=1` with `choice_match`. Follow-up turns need an `auditor_model` o local servers can use the same target. No timestamps are required. Provider/model support for languages, formats and optional parameters varies; unsupported requests are reported as errors rather than silently changing the input. +The file is uploaded as `audio` plus its lowercased extension, so the local +filename is not sent with the upload. ```python from simpleaudit import Auditor, ASRTarget diff --git a/pyproject.toml b/pyproject.toml index 1c3d882..f4c6ac6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -82,6 +82,8 @@ target-version = ["py311", "py312", "py313"] testpaths = ["tests"] addopts = "--cov=simpleaudit --cov-report=html --cov-report=term-missing" asyncio_mode = "auto" +# Tests use plain-http stub hosts; the targets' cleartext warning is expected there. +filterwarnings = ['ignore:target endpoint http\S+ uses plain http:UserWarning'] [tool.coverage.run] source = ["simpleaudit"] diff --git a/simpleaudit/targets/_http.py b/simpleaudit/targets/_http.py new file mode 100644 index 0000000..12d8907 --- /dev/null +++ b/simpleaudit/targets/_http.py @@ -0,0 +1,77 @@ +"""HTTP plumbing shared by the targets: client creation and endpoint provenance. + +``describe_endpoint`` records where a request went (scheme, host, port, and +whether the standard proxy settings route it through a proxy) so an audit +record can say which endpoint answered. It never includes the path, query, +userinfo, headers or keys. +""" + +from __future__ import annotations + +import ipaddress +import urllib.request +import warnings +from typing import Any +from urllib.parse import urlsplit + +_DEFAULT_PORTS = {"http": 80, "https": 443} + + +def make_client(timeout: float) -> Any: + """A fresh ``httpx.AsyncClient(timeout=timeout)``, as the targets always created.""" + import httpx + + return httpx.AsyncClient(timeout=timeout) + + +def _is_loopback(host: str) -> bool: + if host.lower() == "localhost": + return True + try: + return ipaddress.ip_address(host).is_loopback + except ValueError: + return False + + +def describe_endpoint(url: str) -> dict[str, Any]: + """``{"scheme", "host", "port", "via_proxy"}`` for ``url``. + + ``via_proxy`` follows ``urllib.request.getproxies()`` / ``proxy_bypass()`` + (the environment's proxy variables); an injected client's own proxy + configuration is not visible here. + """ + parts = urlsplit(url) + scheme = parts.scheme.lower() + host = parts.hostname or "" + proxies = urllib.request.getproxies() + proxied = bool(proxies.get(scheme) or proxies.get("all")) + return { + "scheme": scheme, + "host": host, + "port": parts.port or _DEFAULT_PORTS.get(scheme), + "via_proxy": proxied and not urllib.request.proxy_bypass(host), + } + + +def warn_if_cleartext(url: str) -> None: + """Warn when ``url`` is plain http to a host other than loopback.""" + parts = urlsplit(url) + if parts.scheme.lower() == "http" and not _is_loopback(parts.hostname or ""): + warnings.warn( + f"target endpoint http://{parts.hostname} uses plain http: requests " + "and responses to it are not encrypted", + UserWarning, + stacklevel=3, + ) + + +def with_endpoint(raw: Any, url: str) -> dict[str, Any]: + """``raw`` with an ``endpoint`` block added; provider fields are never overwritten. + + A dict without an ``endpoint`` key gets one; anything else is wrapped as + ``{"response": raw, "endpoint": ...}``. + """ + endpoint = describe_endpoint(url) + if isinstance(raw, dict) and "endpoint" not in raw: + return {**raw, "endpoint": endpoint} + return {"response": raw, "endpoint": endpoint} diff --git a/simpleaudit/targets/asr.py b/simpleaudit/targets/asr.py index aad3e86..88975a2 100644 --- a/simpleaudit/targets/asr.py +++ b/simpleaudit/targets/asr.py @@ -8,6 +8,7 @@ from typing import Any from urllib.parse import unquote, urlparse +from ._http import make_client, warn_if_cleartext, with_endpoint from .base import TargetContext, TargetResponse @@ -35,6 +36,7 @@ def __init__( ) -> None: self.model = model self.url = base_url.rstrip("/") + "/audio/transcriptions" + warn_if_cleartext(self.url) self.api_key = api_key self.timeout = timeout self.options = dict(options or {}) @@ -63,8 +65,6 @@ async def send( documents: list[str | dict[str, Any]] | None = None, response_format: dict[str, Any] | None = None, ) -> TargetResponse: - import httpx - if file_uri is None: file_uri = next( ( @@ -102,9 +102,7 @@ async def send( headers = {"Authorization": f"Bearer {self.api_key}"} if self.api_key else {} if context: headers.update(context.trace_headers) - client = ( - self._client if self._client is not None else httpx.AsyncClient(timeout=self.timeout) - ) + client = self._client if self._client is not None else make_client(self.timeout) try: with path.open("rb") as audio: response = await client.post( @@ -113,7 +111,8 @@ async def send( headers=headers, files={ "file": ( - path.name, + # Neutral name: the local filename stays local. + "audio" + path.suffix.lower(), audio, mimetypes.guess_type(path.name)[0] or "application/octet-stream", ) @@ -124,7 +123,7 @@ async def send( payload = response.json() if not isinstance(payload, dict) or not isinstance(payload.get("text"), str): raise ValueError("Transcription response must contain a string 'text' field") - return TargetResponse(content=payload["text"], raw=payload) + return TargetResponse(content=payload["text"], raw=with_endpoint(payload, self.url)) finally: if self._client is None: await client.aclose() diff --git a/simpleaudit/targets/decision.py b/simpleaudit/targets/decision.py index 9aa426c..ad30c83 100644 --- a/simpleaudit/targets/decision.py +++ b/simpleaudit/targets/decision.py @@ -40,6 +40,7 @@ from typing import Any from ..context_marks import parse_documents +from ._http import make_client, warn_if_cleartext, with_endpoint from .base import TargetContext, TargetResponse #: Ollama's request-body cap for System One requests without images. @@ -70,6 +71,7 @@ def __init__( max_body_bytes: int | None = None, ) -> None: self.url = url + warn_if_cleartext(url) self.model = model self.headers = dict(headers or {}) if api_key: @@ -177,9 +179,7 @@ async def send( if client := self._client: pass else: - import httpx - - client = httpx.AsyncClient(timeout=self.timeout) + client = make_client(self.timeout) try: resp = await client.post(self.url, content=payload, headers=headers) if resp.status_code >= 400: @@ -204,7 +204,7 @@ async def send( usage = data.get("usage") or {} return TargetResponse( content=f"{choice}: {description}" if description else choice, - raw=data, + raw=with_endpoint(data, self.url), input_tokens=usage.get("input_tokens"), output_tokens=usage.get("output_tokens"), decision=answer, diff --git a/simpleaudit/targets/http.py b/simpleaudit/targets/http.py index 7c38ba3..5bac353 100644 --- a/simpleaudit/targets/http.py +++ b/simpleaudit/targets/http.py @@ -22,6 +22,7 @@ import json from typing import Any, Callable, Dict, List, Optional, Union +from ._http import make_client, warn_if_cleartext, with_endpoint from .base import Target, TargetContext, TargetResponse @@ -67,6 +68,7 @@ def __init__( token_paths: Optional[tuple] = None, ) -> None: self.url = url + warn_if_cleartext(url) self.method = method.upper() self.headers = dict(headers or {}) self.request_template = dict(request_template or {}) @@ -145,9 +147,7 @@ async def send( if client := self._client: pass else: - import httpx - - client = httpx.AsyncClient(timeout=self.timeout) + client = make_client(self.timeout) try: resp = await client.request(self.method, self.url, json=body, headers=headers) resp.raise_for_status() @@ -172,7 +172,7 @@ async def send( return TargetResponse( content=content, - raw=data, + raw=with_endpoint(data, self.url), input_tokens=int(in_tok) if in_tok is not None else None, output_tokens=int(out_tok) if out_tok is not None else None, ) diff --git a/tests/test_endpoint_provenance.py b/tests/test_endpoint_provenance.py new file mode 100644 index 0000000..7e97cbc --- /dev/null +++ b/tests/test_endpoint_provenance.py @@ -0,0 +1,97 @@ +"""Endpoint provenance in target responses, the cleartext warning, neutral ASR filenames.""" + +import warnings + +import httpx +import pytest + +from simpleaudit import ASRTarget, DecisionTarget, HTTPAppTarget +from simpleaudit.targets._http import describe_endpoint + +from .test_decision_target import _context, _endpoint, _send + +URL = "https://user:secret@api.example.com:8443/v1/chat?key=abc123" +PROXY_VARS = ("HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "NO_PROXY") + + +@pytest.fixture +def proxy_env(monkeypatch): + for name in PROXY_VARS: + monkeypatch.delenv(name, raising=False) + monkeypatch.delenv(name.lower(), raising=False) + # An environment proxy for another scheme keeps the system settings out of it. + monkeypatch.setenv("HTTP_PROXY", "http://proxy.test:3128") + return monkeypatch + + +@pytest.mark.parametrize( + "https_proxy,no_proxy,expected", + [(None, None, False), ("http://proxy.test:3128", None, True), + ("http://proxy.test:3128", "api.example.com", False)], +) +def test_describe_endpoint_and_via_proxy(proxy_env, https_proxy, no_proxy, expected): + if https_proxy: + proxy_env.setenv("HTTPS_PROXY", https_proxy) + if no_proxy: + proxy_env.setenv("NO_PROXY", no_proxy) + assert describe_endpoint(URL) == { + "scheme": "https", "host": "api.example.com", "port": 8443, "via_proxy": expected + } + + +def _http_target(payload): + client = httpx.AsyncClient(transport=httpx.MockTransport(lambda r: httpx.Response(200, **payload))) + return HTTPAppTarget(URL, response_path="answer", client=client) + + +@pytest.mark.asyncio +async def test_endpoint_block_is_merged_and_carries_no_path_query_or_key(proxy_env): + raw = (await _http_target({"json": {"answer": "ok", "id": 7}}).send(user="q")).raw + assert raw["answer"] == "ok" and raw["id"] == 7 + assert raw["endpoint"]["host"] == "api.example.com" + assert not any(s in repr(raw["endpoint"]) for s in ("secret", "user", "/v1", "key", "abc123")) + + +@pytest.mark.asyncio +async def test_non_dict_or_conflicting_raw_is_wrapped_not_overwritten(proxy_env): + raw = (await _http_target({"text": "plain"}).send(user="q")).raw + assert raw["response"] == "plain" and raw["endpoint"]["scheme"] == "https" + raw = (await _http_target({"json": {"answer": "ok", "endpoint": "provider"}}).send(user="q")).raw + assert raw["response"]["endpoint"] == "provider" and raw["endpoint"]["port"] == 8443 + + +def test_decision_response_carries_the_endpoint(proxy_env): + client, _ = _endpoint() + raw = _send(DecisionTarget(URL, "clef", client=client), context=_context()).raw + assert raw["answers"]["verdict"]["choice"] == "yes" and raw["endpoint"]["port"] == 8443 + + +def test_plain_http_to_a_remote_host_warns(): + with pytest.warns(UserWarning, match="http://example.com uses plain http"): + HTTPAppTarget("http://example.com/chat") + + +@pytest.mark.parametrize( + "url", ["http://localhost:11434", "http://127.0.0.5/v1", "http://[::1]:8000", "https://example.com"] +) +def test_loopback_and_https_do_not_warn(url): + with warnings.catch_warnings(): + warnings.simplefilter("error") + DecisionTarget(url, "clef") + ASRTarget("m", base_url=url) + + +@pytest.mark.asyncio +async def test_asr_upload_filename_is_neutral(tmp_path): + audio = tmp_path / "Interview Kari Nordmann.WAV" + audio.write_bytes(b"RIFF-audio") + seen = [] + + def handler(request): + seen.append(request.content) + return httpx.Response(200, json={"text": "ok"}) + + async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client: + await ASRTarget("m", client=client).send(user="", file_uri=str(audio)) + assert b'filename="audio.wav"' in seen[0] + assert b"Kari" not in seen[0] and b"Interview" not in seen[0]