feat(evaluations): add LD judge event support

packages/client/src/launchdarkly_ai_server/__init__.py

@@ -22,11 +22,15 @@
     set_conversation_id_if_absent,
 )
 from .evaluations import (
+    Criterion,
+    DatasetRow,
     EvalRunResult,
     EvaluationsError,
     EvaluationsModule,
     GenerationConfig,
+    Judge,
     RunSummary,
+    Scorer,
     init_evaluations,
 )
 from .graph import GraphInstance, graph, resolve_graph
@@ -165,10 +169,14 @@
     "VariationMeta",
     # evaluations
     "EvalRunResult",
+    "Criterion",
+    "DatasetRow",
     "EvaluationsError",
     "EvaluationsModule",
     "GenerationConfig",
+    "Judge",
     "RunSummary",
+    "Scorer",
     "init_evaluations",
     # utils
     "create_handler",

packages/client/src/launchdarkly_ai_server/evaluations/__init__.py

@@ -9,19 +9,24 @@
     Transport,
     urllib_transport,
 )
+from .criteria import Criterion, Judge, Scorer
 from .module import EvaluationsModule, init_evaluations
-from .types import EvalRunResult, GenerationConfig, RunSummary, Usage
+from .types import DatasetRow, EvalRunResult, GenerationConfig, RunSummary, Usage
 
 __all__ = [
     "DEFAULT_BASE_URI",
+    "Criterion",
+    "DatasetRow",
     "EvalRunResult",
     "EvaluationsError",
     "EvaluationsModule",
     "GenerationConfig",
     "HttpResponse",
+    "Judge",
     "LDApiClient",
     "LDApiError",
     "RunSummary",
+    "Scorer",
     "Transport",
     "Usage",
     "init_evaluations",

packages/client/src/launchdarkly_ai_server/evaluations/criteria.py

@@ -0,0 +1,120 @@
+from __future__ import annotations
+
+from collections.abc import Awaitable, Callable
+from dataclasses import dataclass
+from typing import Any
+
+from .types import DatasetRow
+
+type ScorerFn = Callable[[DatasetRow, Any], float | bool | Awaitable[float | bool]]
+
+
+@dataclass(frozen=True)
+class Judge:
+    """Reference to a LaunchDarkly AI Judge config to run for each eval row.
+
+    The SDK does not create or provide built-in judges. Pass the key of a judge
+    that exists in LaunchDarkly. Resolution uses LaunchDarkly flag delivery for
+    the currently served variation.
+    """
+
+    key: str
+    threshold: float | None = None
+    pass_rate_threshold: float | None = None
+    ground_truth_context: str | None = None
+
+    def __post_init__(self) -> None:
+        if not isinstance(self.key, str) or not self.key.strip():
+            raise ValueError("judge key must not be blank")
+        _validate_thresholds(
+            threshold=self.threshold,
+            pass_rate_threshold=self.pass_rate_threshold,
+        )
+
+    @property
+    def criterion_type(self) -> str:
+        return self.key
+
+    def to_criteria_wire(self) -> dict[str, Any]:
+        options = _criteria_options(
+            threshold=self.threshold,
+            pass_rate_threshold=self.pass_rate_threshold,
+            ground_truth_context=self.ground_truth_context,
+        )
+        return {"criterionType": self.criterion_type, "options": options}
+
+
+@dataclass(frozen=True)
+class Scorer:
+    """Local deterministic scorer run for each generated evaluation row.
+
+    ``fn`` may be sync or async and receives ``(row, output)``, where ``row``
+    is the :class:`~launchdarkly_ai_server.evaluations.types.DatasetRow` the
+    output was generated from and ``output`` is the generated output. It must
+    return a boolean or a numeric score from 0 to 1. Boolean results are
+    converted to 1.0 or 0.0 before being emitted as evaluation events.
+
+    ``threshold`` defaults to 1.0: a row passes only on a perfect score, which
+    matches the common case of boolean scorers. Pass a lower threshold for
+    graded numeric scorers.
+    """
+
+    name: str
+    fn: ScorerFn
+    threshold: float | None = 1.0
+    pass_rate_threshold: float | None = None
+
+    def __post_init__(self) -> None:
+        if not isinstance(self.name, str) or not self.name.strip():
+            raise ValueError("scorer name must not be blank")
+        if not callable(self.fn):
+            raise ValueError("scorer fn must be callable")
+        _validate_thresholds(
+            threshold=self.threshold,
+            pass_rate_threshold=self.pass_rate_threshold,
+        )
+
+    @property
+    def criterion_type(self) -> str:
+        return self.name
+
+    def to_criteria_wire(self) -> dict[str, Any]:
+        return {
+            "criterionType": self.criterion_type,
+            "options": _criteria_options(
+                threshold=self.threshold,
+                pass_rate_threshold=self.pass_rate_threshold,
+            ),
+        }
+
+
+type Criterion = Judge | Scorer
+
+
+def _validate_thresholds(
+    *,
+    threshold: float | None,
+    pass_rate_threshold: float | None,
+) -> None:
+    for name, value in (
+        ("threshold", threshold),
+        ("pass_rate_threshold", pass_rate_threshold),
+    ):
+        if value is not None and (value < 0 or value > 1):
+            raise ValueError(f"{name} must be between 0 and 1")
+
+
+def _criteria_options(
+    *,
+    threshold: float | None,
+    pass_rate_threshold: float | None,
+    ground_truth_context: str | None = None,
+) -> dict[str, Any]:
+    options: dict[str, Any] = {}
+    if threshold is not None:
+        options["threshold"] = threshold
+    if pass_rate_threshold is not None:
+        options["passRateThreshold"] = pass_rate_threshold
+    if ground_truth_context is not None:
+        options["groundTruthContext"] = ground_truth_context
+    return options

packages/client/src/launchdarkly_ai_server/evaluations/events.py

@@ -0,0 +1,110 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import StrEnum
+from typing import Any
+
+
+class EvaluationStatus(StrEnum):
+    COMPLETE = "COMPLETE"
+    ERROR = "ERROR"
+
+
+class EvaluationEventKind(StrEnum):
+    JUDGE = "judge"
+    SCORER = "scorer"
+
+
+@dataclass(frozen=True)
+class TokenUsage:
+    """Token usage reported by an LD Judge provider call."""
+
+    input_tokens: int
+    output_tokens: int
+
+    def to_wire(self) -> dict[str, int]:
+        return {
+            "inputTokens": self.input_tokens,
+            "outputTokens": self.output_tokens,
+        }
+
+
+@dataclass(frozen=True, kw_only=True)
+class EvaluationEventPayload:
+    """Common fields emitted for every SDK-run evaluation criterion result."""
+
+    project_key: str
+    evaluation_id: str
+    evaluation_run_id: str
+    run_id: str
+    dataset_id: str
+    row_index: int
+    criterion_type: str
+    kind: EvaluationEventKind
+    event_id: str
+    emitted_at: str
+    evaluation_key: str
+    dataset_key: str
+    status: EvaluationStatus
+    started_at: str
+    evaluated_at: str
+    latency_ms: int
+    evaluation_version: int | None = None
+    score: float | None = None
+    reason: str | None = None
+    error: dict[str, Any] | None = None
+    error_message: str | None = None
+
+    def to_track_payload(self) -> dict[str, Any]:
+        payload: dict[str, Any] = {
+            "projectKey": self.project_key,
+            "evaluationId": self.evaluation_id,
+            "evaluationRunId": self.evaluation_run_id,
+            "runId": self.run_id,
+            "datasetId": self.dataset_id,
+            "rowIndex": self.row_index,
+            "criterionType": self.criterion_type,
+            "kind": self.kind.value,
+            "eventId": self.event_id,
+            "emittedAt": self.emitted_at,
+            "evaluationKey": self.evaluation_key,
+            "evaluationVersion": self.evaluation_version,
+            "datasetKey": self.dataset_key,
+            "status": self.status.value,
+            "startedAt": self.started_at,
+            "evaluatedAt": self.evaluated_at,
+            "latencyMs": self.latency_ms,
+            "score": self.score,
+            "reason": self.reason,
+            "error": self.error,
+            "errorMessage": self.error_message,
+        }
+        return {key: value for key, value in payload.items() if value is not None}
+
+
+@dataclass(frozen=True, kw_only=True)
+class LDJudgeEvaluationEventPayload(EvaluationEventPayload):
+    """Payload for one LaunchDarkly AI Judge result on one dataset row."""
+
+    kind: EvaluationEventKind = EvaluationEventKind.JUDGE
+    judge_key: str
+    variation_key: str
+    version: int | None = None
+    usage: TokenUsage | None = None
+
+    def to_track_payload(self) -> dict[str, Any]:
+        payload = super().to_track_payload()
+        payload["judgeKey"] = self.judge_key
+        payload["variationKey"] = self.variation_key
+        if self.version is not None:
+            payload["version"] = self.version
+        if self.usage is not None:
+            payload["usage"] = self.usage.to_wire()
+        return payload
+
+
+@dataclass(frozen=True, kw_only=True)
+class DeterministicScorerEvaluationEventPayload(EvaluationEventPayload):
+    """Payload for one local deterministic scorer result on one dataset row."""
+
+    kind: EvaluationEventKind = EvaluationEventKind.SCORER