From 741f773ee1c482e48e7579f10e92b4c53572022c Mon Sep 17 00:00:00 2001 From: Jeff Dupont Date: Tue, 20 Oct 2026 15:33:24 -0700 Subject: [PATCH 1/6] [AIC-3500] chore(AIC-3500): commit an API report for every public module Each public module (every package root, and each module of the core package's experimental subpackage) now has a committed report at packages//api-report/.txt, generated by scripts/api_report.py with griffe. A report lists each name in __all__ with its full signature, and a class's public members indented below it. tests/test_api_report.py fails while any report is out of date, so `make test` and CI catch a public name or signature change whose report wasn't regenerated. It also checks the generator against a fixture package. `make api-report` writes the reports, `make api-report-check` checks them, and .github/CODEOWNERS gives them to team-ai-agents. griffelib is now an explicit dev dependency. Co-Authored-By: Claude Opus 5.5 --- .github/CODEOWNERS | 2 + AGENTS.md | 8 + Makefile | 10 +- .../launchdarkly_ai_claude_agents.txt | 18 ++ .../launchdarkly_ai_claude_messages.txt | 3 + ...kly_ai_server.experimental.evaluations.txt | 72 ++++++ ...chdarkly_ai_server.experimental.skills.txt | 81 ++++++ .../api-report/launchdarkly_ai_server.txt | 236 ++++++++++++++++++ .../launchdarkly_ai_langchain_agents.txt | 5 + .../launchdarkly_ai_langchain_messages.txt | 3 + .../launchdarkly_ai_openai_agents.txt | 6 + .../launchdarkly_ai_openai_messages.txt | 3 + .../launchdarkly_ai_opencode_agents.txt | 4 + pyproject.toml | 1 + scripts/api_report.py | 184 ++++++++++++++ tests/test_api_report.py | 141 +++++++++++ uv.lock | 1 + 17 files changed, 777 insertions(+), 1 deletion(-) create mode 100644 .github/CODEOWNERS create mode 100644 packages/claude-agents/api-report/launchdarkly_ai_claude_agents.txt create mode 100644 packages/claude-messages/api-report/launchdarkly_ai_claude_messages.txt create mode 100644 packages/client/api-report/launchdarkly_ai_server.experimental.evaluations.txt create mode 100644 packages/client/api-report/launchdarkly_ai_server.experimental.skills.txt create mode 100644 packages/client/api-report/launchdarkly_ai_server.txt create mode 100644 packages/langchain-agents/api-report/launchdarkly_ai_langchain_agents.txt create mode 100644 packages/langchain-messages/api-report/launchdarkly_ai_langchain_messages.txt create mode 100644 packages/openai-agents/api-report/launchdarkly_ai_openai_agents.txt create mode 100644 packages/openai-messages/api-report/launchdarkly_ai_openai_messages.txt create mode 100644 packages/opencode-agents/api-report/launchdarkly_ai_opencode_agents.txt create mode 100644 scripts/api_report.py create mode 100644 tests/test_api_report.py diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 00000000..9dc72f1d --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,2 @@ +# The API reports are the public API contract: a change to one changes a public name or signature. +/packages/*/api-report/ @launchdarkly/team-ai-agents diff --git a/AGENTS.md b/AGENTS.md index d022c77f..661c7b7d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -74,6 +74,14 @@ them mypy-clean under `strict` is a separate piece of work. The `UP046` rule (PEP 695 `type` statement syntax for generic classes) is **ignored** in `pyproject.toml`. Do not convert `class Foo(Generic[T]):` to PEP 695 syntax — it is a deliberate migration decision. +## Public API Reports + +Every public module has a committed API report at `packages//api-report/.txt`: each name in its `__all__`, one per line, with its full signature, and a class's public members indented below it. The reports are the public API contract. + +- **Change a public name or signature, then run `make api-report`** and commit the updated report in the same PR. `make test` fails while a report is out of date (`tests/test_api_report.py`), and so does `make api-report-check`. +- **The reports are owned by `@launchdarkly/team-ai-agents`** in `.github/CODEOWNERS`, so their approval of the report diff is how public API changes are reviewed. +- **Never edit a report by hand.** It is generated by `scripts/api_report.py` with griffe. + ## PR Titles and the Changelog PRs are squash-merged with the PR title as the commit message, and release-please builds each package's changelog from those messages. diff --git a/Makefile b/Makefile index 71630ad9..82de8e77 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: start test typecheck lint format format-check lint-fix build clean +.PHONY: start test typecheck lint format format-check lint-fix build clean api-report api-report-check # Run main.py (uv run python main.py) start: @@ -17,6 +17,14 @@ test: typecheck: uv run mypy packages/*/src +# Write every package's API report (packages/*/api-report/); commit the result +api-report: + uv run python scripts/api_report.py + +# Fail if any API report is out of date (also run by `make test`) +api-report-check: + uv run python scripts/api_report.py --check + # Check for lint errors lint: uv run ruff check . diff --git a/packages/claude-agents/api-report/launchdarkly_ai_claude_agents.txt b/packages/claude-agents/api-report/launchdarkly_ai_claude_agents.txt new file mode 100644 index 00000000..f7a24162 --- /dev/null +++ b/packages/claude-agents/api-report/launchdarkly_ai_claude_agents.txt @@ -0,0 +1,18 @@ +# Public API of launchdarkly_ai_claude_agents. Generated by `make api-report`; do not edit. +build_prompt(config: AiConfigRep, user_input: str | None, variables: dict[str, Any], history: list[dict[str, Any]] | None = None) -> tuple[str, str | None] +build_tool_mcp(config_tools: dict[str, Any], handlers: dict[str, Any]) -> Any [async] +claude_agents(config_key: str, user_input: str, context: LDContext, **kwargs: Any) -> Any +claude_graph(key: str, **options: Any) -> Any +ClaudeBash = NativeTool('Bash') +ClaudeEdit = NativeTool('Edit') +ClaudeGlob = NativeTool('Glob') +ClaudeGrep = NativeTool('Grep') +ClaudeNotebookEdit = NativeTool('NotebookEdit') +ClaudeRead = NativeTool('Read') +ClaudeTodoWrite = NativeTool('TodoWrite') +ClaudeWebFetch = NativeTool('WebFetch') +ClaudeWebSearch = NativeTool('WebSearch') +ClaudeWrite = NativeTool('Write') +create_claude_agents_handler(*, capture_content: bool | None = None) -> ProviderHandler +partition_tools(config_tools: dict[str, Any] | None, tool_handlers: dict[str, Any]) -> tuple[dict[str, Any], dict[str, Any], list[str]] +to_claude_agents(def_promise: Any, opts: dict[str, Any] | None = None) -> Any diff --git a/packages/claude-messages/api-report/launchdarkly_ai_claude_messages.txt b/packages/claude-messages/api-report/launchdarkly_ai_claude_messages.txt new file mode 100644 index 00000000..b051ead8 --- /dev/null +++ b/packages/claude-messages/api-report/launchdarkly_ai_claude_messages.txt @@ -0,0 +1,3 @@ +# Public API of launchdarkly_ai_claude_messages. Generated by `make api-report`; do not edit. +claude_messages(config_key: str, user_input: str, context: LDContext, **kwargs: Any) -> Any +create_claude_messages_handler(*, capture_content: bool | None = None) -> ProviderHandler diff --git a/packages/client/api-report/launchdarkly_ai_server.experimental.evaluations.txt b/packages/client/api-report/launchdarkly_ai_server.experimental.evaluations.txt new file mode 100644 index 00000000..b4abf15e --- /dev/null +++ b/packages/client/api-report/launchdarkly_ai_server.experimental.evaluations.txt @@ -0,0 +1,72 @@ +# Public API of launchdarkly_ai_server.experimental.evaluations. Generated by `make api-report`; do not edit. +AIConfig [class, @dataclass(frozen=True)] + __init__(key: str, variation: str) + key: str + variation: str +Criterion = Judge | Scorer [type] +DatasetRow [class, @dataclass] + __init__(row_index: int, input: str | None = None, expected_output: str | None = None, variables: dict[str, Any] = dict(), metadata: dict[str, Any] | None = None) + expected_output: str | None = None + input: str | None = None + metadata: dict[str, Any] | None = None + row_index: int + variables: dict[str, Any] = field(default_factory=dict) +EvalRunResult [class, @dataclass] + __init__(passed: bool, url: str, run_id: str, summary: RunSummary) + passed: bool + run_id: str + summary: RunSummary + url: str +EvalTool [class, @dataclass(frozen=True)] + __init__(key: str, implementation: ToolImplementation, schema: dict[str, Any], description: str = '') + description: str = '' + implementation: ToolImplementation + key: str + project_key: str | None = field(default=None, init=False) + schema: dict[str, Any] + source: Literal['library', 'inline'] = field(default='inline', init=False) + to_create_wire() -> dict[str, Any] + version: int | None = field(default=None, init=False) +EvaluationsError(Exception) [class] +EvaluationsModule [class] + __init__(api_client: LDApiClient, project_key: str, sdk_key: str | None, ui_base_uri: str = DEFAULT_UI_BASE_URI) + api: LDApiClient [property] + project_key: str [property] + run(*, key: str, dataset: str | Sequence[InlineDatasetRow], handler: EvalHandler, generation: GenerationConfig | None = None, ai_config: AIConfig | None = None, tools: Sequence[EvalTool] | None = None, criteria: list[Criterion] | None = None, judge_handlers: list[EvalHandler] | None = None, concurrency: int = 10, poll_interval_seconds: float | None = None, poll_timeout_seconds: float | None = None) -> EvalRunResult [async] + sdk_key: str | None [property] + tools: ToolsClient [property] + ui_base_uri: str [property] +GenerationConfig(TypedDict) [class] + instructions: str + messages: list[dict[str, Any]] + model: str + output_format: dict[str, Any] + parameters: dict[str, Any] + prompt_snippets: dict[str, str] + provider: str +init_evaluations(project_key: str | None = None, api_key: str | None = None, sdk_key: str | None = None, base_uri: str | None = None, ui_base_uri: str | None = None, transport: Transport = urllib_transport) -> EvaluationsModule +Judge [class, @dataclass(frozen=True)] + __init__(key: str, threshold: float | None = DEFAULT_JUDGE_THRESHOLD, pass_rate_threshold: float | None = None, ground_truth_context: str | None = None) + criterion_type: str [property] + ground_truth_context: str | None = None + key: str + pass_rate_threshold: float | None = None + threshold: float | None = DEFAULT_JUDGE_THRESHOLD + to_criteria_wire() -> dict[str, Any] +RunSummary [class, @dataclass] + __init__(total_rows: int = 0, passed_rows: int = 0, failed_rows: int = 0, error_rows: int = 0, pending_rows: int = 0) + error_rows: int = 0 + failed_rows: int = 0 + from_wire(data: Mapping[str, Any] | None) -> RunSummary [classmethod] + passed_rows: int = 0 + pending_rows: int = 0 + total_rows: int = 0 +Scorer [class, @dataclass(frozen=True)] + __init__(name: str, fn: ScorerFn, threshold: float | None = 1.0, pass_rate_threshold: float | None = None, success_direction: SuccessDirection = 'higher_is_better') + criterion_type: str [property] + fn: ScorerFn + name: str + pass_rate_threshold: float | None = None + success_direction: SuccessDirection = 'higher_is_better' + threshold: float | None = 1.0 + to_criteria_wire() -> dict[str, Any] diff --git a/packages/client/api-report/launchdarkly_ai_server.experimental.skills.txt b/packages/client/api-report/launchdarkly_ai_server.experimental.skills.txt new file mode 100644 index 00000000..18639cd0 --- /dev/null +++ b/packages/client/api-report/launchdarkly_ai_server.experimental.skills.txt @@ -0,0 +1,81 @@ +# Public API of launchdarkly_ai_server.experimental.skills. Generated by `make api-report`; do not edit. +all_skills() -> list[Skill] [async] +FDv2SkillStore [class] + __init__(sdk_key: str, *, base_uri: str | None = None, stream_uri: str | None = None, mode: Mode = 'stream', poll_interval: float = 30.0, read_timeout: float | None = None, initial_backoff: float = 1.0, max_backoff: float = 30.0, _requester: Any = None) + add_listener(kind: str, fn: Callable[[dict[str, Any]], Any]) -> None + all_objects(kind: str) -> dict[str, dict[str, Any]] + close(timeout: float = 5.0) -> None + diagnostics: StoreDiagnostics [property] + failed: str | None [property] + get_object(kind: str, key: str, version: int | None = None) -> dict[str, Any] | None + is_initialized() -> bool + remove_listener(kind: str, fn: Callable[[dict[str, Any]], Any]) -> None + start() -> FDv2SkillStore + wait_for_skills(timeout: float = 10.0) -> bool +get_skill(key: str, *, version: int | None = None) -> Skill | None [async] +get_skill_result(key: str, *, version: int | None = None) -> SkillOutcome [async] +get_skills(refs: Sequence[SkillReference | str]) -> list[Skill] [async] +InMemorySkillStore [class] + __init__(objects: dict[str, dict[str, Any]] | None = None) + add_listener(kind: str, fn: Callable[[dict[str, Any]], Any]) -> None + all_objects(kind: str) -> dict[str, dict[str, Any]] + get_object(kind: str, key: str, version: int | None = None) -> dict[str, Any] | None + put(raw: dict[str, Any]) -> None + remove_listener(kind: str, fn: Callable[[dict[str, Any]], Any]) -> None +MANIFEST_FILENAME = '.launchdarkly-skills.json' +MANIFEST_VERSION = 1 +OnUnavailable = Literal['keep', 'raise'] +ReconcileAction [class, @dataclass(frozen=True)] + __init__(key: str, action: ReconcileActionKind, version: int | None = None, path: str | None = None, error: str | None = None) + action: ReconcileActionKind + error: str | None = None + key: str + path: str | None = None + version: int | None = None +ReconcileActionKind = Literal['written', 'updated', 'skipped_current', 'removed', 'error'] +ReconcileReport [class, @dataclass(frozen=True)] + __init__(actions: list[ReconcileAction] = list()) + actions: list[ReconcileAction] = field(default_factory=list) + errors: list[ReconcileAction] [property] + ok: bool [property] +set_skill_store(store: SkillStore | None) -> None +Skill [class, @dataclass(frozen=True)] + __init__(key: str, version: int, content: bytes, content_hash: str, name: str | None = None, description: str | None = None) + content: bytes + content_hash: str + description: str | None = None + key: str + name: str | None = None + version: int +SKILL_FILENAME = 'SKILL.md' +skill_refs(config: AiConfigRep | None) -> list[SkillReference] +SkillOutcome [class, @dataclass(frozen=True)] + __init__(skill: Skill | None, reason: SkillOutcomeReason, detail: str | None) + detail: str | None + reason: SkillOutcomeReason + skill: Skill | None +SkillOutcomeReason = Literal['absent', 'integrity_failure', 'ok', 'store_unavailable', 'wrong_version'] +SkillReference [class, @dataclass(frozen=True)] + __init__(key: str, version: int) + key: str + version: int +SkillStore(Protocol) [class] + all_objects(kind: str) -> dict[str, dict[str, Any]] + get_object(kind: str, key: str, version: int | None = None) -> dict[str, Any] | None +SkillWatcher [class] + __init__(request: Sequence[Skill | SkillReference | str] | str, root: str | os.PathLike[str], store: Any, *, prune: bool, timeout: float, on_unavailable: OnUnavailable, debounce: float, on_reconcile: Callable[[ReconcileReport], Any] | None) + close(timeout: float = 15.0) -> None + notify(_raw: Any = None) -> None + reconciles: int [property] +StoreDiagnostics [class, @dataclass] + __init__(payloads_transferred: int = 0, skill_objects_received: int = 0, objects_ignored: int = 0, objects_revoked: int = 0, payloads_ignored: int = 0, hashless_objects: int = 0, connection_failures: int = 0, last_error: str | None = None) + connection_failures: int = 0 + hashless_objects: int = 0 + last_error: str | None = None + objects_ignored: int = 0 + objects_revoked: int = 0 + payloads_ignored: int = 0 + payloads_transferred: int = 0 + skill_objects_received: int = 0 +watch_skills(skills: Sequence[Skill | SkillReference | str] | str, root: str | os.PathLike[str], *, prune: bool = True, timeout: float = 10.0, on_unavailable: OnUnavailable = 'keep', debounce: float = DEFAULT_DEBOUNCE_SECONDS, on_reconcile: Callable[[ReconcileReport], Any] | None = None) -> tuple[ReconcileReport, SkillWatcher] [async] +write_skills(skills: Sequence[Skill | SkillReference | str] | str, root: str | os.PathLike[str], *, prune: bool = True, timeout: float = 10.0, on_unavailable: OnUnavailable = 'keep') -> ReconcileReport [async] diff --git a/packages/client/api-report/launchdarkly_ai_server.txt b/packages/client/api-report/launchdarkly_ai_server.txt new file mode 100644 index 00000000..ecc9ea06 --- /dev/null +++ b/packages/client/api-report/launchdarkly_ai_server.txt @@ -0,0 +1,236 @@ +# Public API of launchdarkly_ai_server. Generated by `make api-report`; do not edit. +add_cached_tokens_to_input(raw_usage: dict[str, Any]) -> SpanUsage +AiConfigRep = dict[str, Any] +any_multimodal(turns: list[dict[str, Any]]) -> bool +build_judge_tasks(*, config: AiConfigRep, user_context: LDContext, handler: ProviderHandler, handlers: list[ProviderHandler] | None = None, llm_response: str, base_track_data: TrackData, user_input: str | None = None, trajectory: str = '') -> list[JudgeTask] [async] +compose(a: Registry, b: Registry) -> Registry +compose_history(*, history: list[dict[str, Any]], user_input: str | None = None, config_messages: list[dict[str, Any]] | None = None) -> list[dict[str, Any]] +config(*, key: str, handler: ProviderHandler | list[ProviderHandler] | None = None, tool_handlers: dict[str, Callable[..., Any] | NativeTool] | None = None, registry: Any = None, skip_judges: bool = False) -> ConfigInstance +ConfigInstance [class] + __init__(key: str, handler: ProviderHandler | list[ProviderHandler] | None, tool_handlers: dict[str, Callable[..., Any] | NativeTool] | None, registry: Any, skip_judges: bool = False) + invoke(user_input: str | None, context: LDContext, variables: dict[str, Any] | None = None, history: list[dict[str, Any]] | None = None) -> ProviderResponse[Any] [async] + stream(user_input: str | None, context: LDContext, variables: dict[str, Any] | None = None, history: list[dict[str, Any]] | None = None) -> AsyncGenerator[StreamEvent, None] +configure(*, options: SDKOptions) -> None +content_to_text(content: MessageContent) -> str +conversation_id(conversation: str | None) -> Iterator[None] [@contextmanager] +ConversationIdSpanProcessor(_SpanProcessorBase) [class] + force_flush(timeout_millis: int | None = None) -> bool + on_end(span: Any) -> None + on_start(span: Span, parent_context: otel_context.Context | None = None) -> None + shutdown() -> None +create_handler(provides_for: tuple[str, Literal['agent', 'messages']], fn: _HandlerFn, stream_fn: _StreamFn | None = None, capture_content: bool | None = None) -> ProviderHandler +create_run_usage() -> RunUsage +end_span_once(span: Any, tracker: set[int], abandoned: bool = False, cancelled: bool = False) -> None +end_unfinished_spans(*spans: Any) -> None +execute_and_stream(*, config_key: str, config: AiConfigRep, meta: VariationMeta, user_context: LDContext, handler: ProviderHandler, user_input: str | None = None, tool_handlers: dict[str, Callable[..., Any] | NativeTool] | None = None, variables: dict[str, Any] | None = None, graph_key: str | None = None, history: list[dict[str, Any]] | None = None) -> AsyncGenerator[ExecuteStreamEvent, None] [async] +execute_and_track(*, config_key: str, config: AiConfigRep, meta: VariationMeta, user_context: LDContext, handler: ProviderHandler, user_input: str | None = None, tool_handlers: dict[str, Callable[..., Any] | NativeTool] | None = None, variables: dict[str, Any] | None = None, graph_key: str | None = None, history: list[dict[str, Any]] | None = None) -> dict[str, Any] [async] +ExecuteStreamDoneEvent = dict[str, Any] +ExecuteStreamEvent = dict[str, Any] +extract_variation(config_key: str, context: dict[str, Any]) -> dict[str, Any] [async] +get_client() -> Any +global_registry = Registry() +graph(key: str, *, handlers: list[ProviderHandler] | None = None, tool_handlers: dict[str, Callable[..., Any] | NativeTool] | None = None, registry: Any = None, graph_judge: str | None = None) -> GraphInstance +GraphArgs = dict[str, Any] +GraphDefinition [class] + __init__(*, key: str, enabled: bool, root: GraphNode | None, get_node: Callable[[str], GraphNode | None], get_child_nodes: Callable[[str], list[GraphNode]], get_parent_nodes: Callable[[str], list[GraphNode]], terminal_nodes: Callable[[], list[GraphNode]], is_terminal: Callable[[str], bool], edges_from: Callable[[str], list[GraphEdge]], run_node: Callable[..., Any], route: Callable[..., Any], traverse: Callable[..., Any], reverse_traverse: Callable[..., Any]) + edges_from = edges_from + enabled = enabled + get_child_nodes = get_child_nodes + get_node = get_node + get_parent_nodes = get_parent_nodes + is_terminal = is_terminal + key = key + reverse_traverse = reverse_traverse + root = root + route = route + run_node = run_node + terminal_nodes = terminal_nodes + traverse = traverse +GraphEdge [class, @dataclass] + __init__(key: str, source_key: str, target_key: str, handoff: dict[str, Any] | None = None) + handoff: dict[str, Any] | None = None + key: str + source_key: str + target_key: str +GraphInstance [class] + __init__(key: str, options: dict[str, Any]) + invoke(user_input: str | None, context: LDContext, variables: dict[str, Any] | None = None, history: list[dict[str, Any]] | None = None) -> ProviderGraphResponse [async] + stream(user_input: str | None, context: LDContext, variables: dict[str, Any] | None = None, history: list[dict[str, Any]] | None = None) -> AsyncGenerator[GraphStreamEvent, None] +GraphNode [class, @dataclass] + __init__(key: str, config: AiConfigRep, meta: VariationMeta, edges: list[GraphEdge] = list(), is_terminal: bool = False) + config: AiConfigRep + edges: list[GraphEdge] = field(default_factory=list) + is_terminal: bool = False + key: str + meta: VariationMeta +GraphOptions = dict[str, Any] +GraphStreamEvent = dict[str, Any] +GraphTopology = dict[str, Any] +HandlerResult = dict[str, Any] +HandlerStreamEvent [class] +has_multimodal_content(content: MessageContent) -> bool +image_block_to_url(block: dict[str, Any]) -> str +init_client(options: InitClientOptions | None = None, client: Any = None) -> Any [async] +InitClientOptions = dict[str, Any] +InputTokenDetails [class, @dataclass] + __init__(uncached: int = 0, cache_read: int = 0, cache_creation: int = 0) + cache_creation: int = 0 + cache_read: int = 0 + uncached: int = 0 +inspect_config(config_key: str, context: dict[str, Any]) -> dict[str, Any] [async] +is_content_blocks(content: MessageContent) -> bool +JudgeResult [class, @dataclass] + __init__(usage: UsageDict, response: str, score: float) + response: str + score: float + usage: UsageDict +JudgeRunResult [class, @dataclass] + __init__(score: float, response: str, usage: UsageDict, track_data: TrackData) + response: str + score: float + track_data: TrackData + usage: UsageDict +JudgeTask [class, @dataclass] + __init__(config_key: str, judge_config: AiConfigRep, judge_meta: VariationMeta, actual_output: str, user_context: LDContext, judge_provider: str | None, judge_mode: str, collapse_messages: bool, parent_track_data: TrackData, variables: dict[str, Any] | None = None, evaluation_metric_key: str | None = None, user_input: str | None = None, trajectory: str = '') + actual_output: str + collapse_messages: bool + config_key: str + evaluation_metric_key: str | None = None + judge_config: AiConfigRep + judge_meta: VariationMeta + judge_mode: str + judge_provider: str | None + parent_track_data: TrackData + trajectory: str = '' + user_context: LDContext + user_input: str | None = None + variables: dict[str, Any] | None = None +lang_chain_content_text(content: Any) -> str +lang_chain_finish_reasons(source: Any) -> list[str] | None +lang_chain_span_messages(messages: list[Any]) -> tuple[str | None, list[SpanMessage]] +lang_chain_span_usage(usage: dict[str, Any] | None) -> SpanUsage | None +LDClientInterface [class] + close() -> None [async] + flush() -> None [async] + track(event_name: str, context: LDContext, data: Any = None, metric_value: float | None = None) -> None + variation(key: str, context: LDContext, default_value: Any) -> Any [async] +LDContext = dict[str, Any] +make_track_data(node: GraphNode, graph_key: str, run_id: str) -> dict[str, Any] +Message [class] + content: str + role: Literal['user', 'assistant', 'system'] +model_stamps_from_meta(meta: Any) -> dict[str, Any] +NATIVE_TOOL_KEY: str = '__ld_native_tool__' +NativeTool [class] + __init__(tool_name: str) + id: object = object() + tool_name: str = tool_name +normalize_mode(mode: str | None) -> Literal['agent', 'messages'] +number_or_zero(value: Any) -> int +omit_model_stamps(track_data: Mapping[str, Any]) -> dict[str, Any] +parse_ai_config(raw: Any) -> ParseResult +parse_json_with_possible_fences(raw_text: str) -> Any | None +parse_template(template: str, variables: dict[str, Any]) -> str +parse_usage(usage: dict[str, Any]) -> dict[str, Any] +ParseFailure [class, @dataclass] + __init__(success: Literal[False], error: dict[str, str]) + error: dict[str, str] + success: Literal[False] +ParseResult = ParseSuccess[Any] | ParseFailure +ParseSuccess(Generic[T]) [class, @dataclass] + __init__(success: Literal[True], data: T) + data: T + success: Literal[True] +ProviderGraphResponse [class, @dataclass] + __init__(response: str, usage: UsageDict, judge_results: dict[str, JudgeResult] | None = None) + judge_results: dict[str, JudgeResult] | None = None + response: str + usage: UsageDict +ProviderHandler [class] + __init__(fn: _HandlerFn, provides_for: tuple[str, Literal['agent', 'messages']] | None = None, stream_fn: _StreamFn | None = None, capture_content: bool | None = None) + capture_content: bool [property] + has_stream: bool [property] + provides_for: tuple[str, Literal['agent', 'messages']] | None = provides_for + stream(config: AiConfigRep, user_input: str | None = None, tool_handlers: dict[str, Any] | None = None, variables: dict[str, Any] | None = None, history: list[dict[str, Any]] | None = None) -> AsyncGenerator[dict[str, Any], None] [async] +ProviderResponse(Generic[T]) [class, @dataclass] + __init__(response: T, usage: UsageDict, judge_results: dict[str, JudgeResult] | None = None, judge_tasks: list[JudgeTask] | None = None, track_data: TrackData | None = None) + judge_results: dict[str, JudgeResult] | None = None + judge_tasks: list[JudgeTask] | None = None + response: T + track_data: TrackData | None = None + usage: UsageDict +register_ai_sdk_package(name: str, version: str) -> None +Registry [class] + __init__(*, handlers: list[ProviderHandler] | None = None, tools: dict[str, Callable[..., Any] | NativeTool] | None = None) + handlers: list[ProviderHandler] [property] + register(*, handlers: list[ProviderHandler] | None = None, tools: dict[str, Callable[..., Any] | NativeTool] | None = None) -> None + tools: dict[str, Callable[..., Any] | NativeTool] [property] +report_usage(helper: str, package_name: str | None = None, package_version: str | None = None) -> None +resolve_graph(key: str, *, context: LDContext, handlers: list[ProviderHandler] | None = None, tool_handlers: dict[str, Callable[..., Any] | NativeTool] | None = None, registry: Any = None) -> GraphDefinition [async] +resolve_handlers(registry: Registry | None, local_handlers: list[ProviderHandler] | None) -> list[ProviderHandler] | None +resolve_tools(registry: Registry | None, local_tools: dict[str, Callable[..., Any] | NativeTool] | None) -> dict[str, Callable[..., Any] | NativeTool] | None +run_judge(task: JudgeTask, handlers: list[ProviderHandler]) -> JudgeRunResult | None [async] +run_judges(*, config: AiConfigRep, user_context: LDContext, handler: ProviderHandler, handlers: list[ProviderHandler] | None = None, user_input: str | None, llm_response: str, base_track_data: TrackData, tool_handlers: dict[str, Callable[..., Any] | NativeTool] | None = None, graph_key: str | None = None, trajectory: str = '') -> dict[str, JudgeResult] [async] +RunUsage [class, @dataclass] + __init__(total: SpanUsage = SpanUsage(), _turns: int = 0) + add(turn: SpanUsage | None) -> None + reported: bool [property] + total: SpanUsage = field(default_factory=SpanUsage) +SDK_INFO_CONTEXT: LDContext = {'kind': 'ld_ai', 'key': 'ld-internal-tracking', 'anonymous': True} +SDK_INFO_EVENT = '$ld:ai:sdk:info' +SDKOptions [class, @dataclass(frozen=True)] + __init__(capture_content: bool = False) + capture_content: bool = False +set_conversation_id_if_absent(span: Any, conversation: str) -> None +set_input_content_attributes(span: Any, capture: bool, *, system_instructions: str | None = None, messages: list[SpanMessage] | None = None, tool_definitions: list[ToolDefinitionInput] | None = None) -> None +set_ld_span_attributes(span: Any, variables: dict[str, Any] | None) -> None +set_model_identity_attributes(span: Any, provider_name: str, request_model: str, legacy_system: str | None = None) -> None +set_output_content_attributes(span: Any, capture: bool, messages: list[SpanMessage]) -> None +set_tool_call_content_attributes(span: Any, capture: bool, *, arguments: Any = None, result: Any = None) -> None +set_tool_definition_attributes(span: Any, capture: bool, tools: list[ToolDefinitionInput]) -> None +set_usage_span_attributes(span: Any, usage: SpanUsage) -> None +shutdown() -> None [async] +SpanMessage [class, @dataclass] + __init__(role: str, parts: list[SpanMessagePart] = list(), finish_reason: str | None = None) + finish_reason: str | None = None + parts: list[SpanMessagePart] = field(default_factory=list) + role: str + to_canonical() -> dict[str, Any] + to_text() -> str +SpanMessagePart [class, @dataclass] + __init__(type: Literal['text', 'reasoning', 'tool_call', 'tool_call_response'], content: str | None = None, id: str | None = None, name: str | None = None, arguments: Any = None, result: Any = None) + arguments: Any = None + content: str | None = None + id: str | None = None + name: str | None = None + result: Any = None + to_canonical() -> dict[str, Any] + to_text() -> str + type: Literal['text', 'reasoning', 'tool_call', 'tool_call_response'] +SpanUsage [class, @dataclass] + __init__(input: int = 0, output: int = 0, cache_read: int = 0, cache_creation: int = 0) + cache_creation: int = 0 + cache_read: int = 0 + input: int = 0 + output: int = 0 +StreamChunkEvent = dict[str, Any] +StreamDoneEvent = dict[str, Any] +StreamEvent = dict[str, Any] +text_message(role: str, content: str) -> SpanMessage +to_ld_context(client: Any, context: dict[str, Any]) -> Any +to_semconv_finish_reason(raw: str | None) -> str | None +to_usage_dict(usage: dict[str, Any]) -> UsageDict +ToolDefinitionInput [class, @dataclass] + __init__(name: str, description: str | None = None, parameters: Any = None) + description: str | None = None + name: str + parameters: Any = None +TrackData = dict[str, Any] +UsageDict [class, @dataclass] + __init__(input: int = 0, output: int = 0, total: int = 0, input_details: InputTokenDetails | None = None) + input: int = 0 + input_details: InputTokenDetails | None = None + output: int = 0 + total: int = 0 +VariationMeta = dict[str, Any] +wrap_tool_handlers(tool_handlers: dict[str, Callable[..., Any] | NativeTool] | None, user_context: LDContext, track_data: TrackData) -> dict[str, Callable[..., Any]] diff --git a/packages/langchain-agents/api-report/launchdarkly_ai_langchain_agents.txt b/packages/langchain-agents/api-report/launchdarkly_ai_langchain_agents.txt new file mode 100644 index 00000000..4a23ab21 --- /dev/null +++ b/packages/langchain-agents/api-report/launchdarkly_ai_langchain_agents.txt @@ -0,0 +1,5 @@ +# Public API of launchdarkly_ai_langchain_agents. Generated by `make api-report`; do not edit. +create_langchain_agents_handler(llm: Any = None, *, capture_content: bool | None = None) -> ProviderHandler +langchain_agents(config_key: str, user_input: str, context: LDContext, **kwargs: Any) -> Any +langchain_graph(key: str, llm: Any = None, **options: Any) -> Any +to_lang_graph(def_promise: Any, opts: dict[str, Any] | None = None) -> Any diff --git a/packages/langchain-messages/api-report/launchdarkly_ai_langchain_messages.txt b/packages/langchain-messages/api-report/launchdarkly_ai_langchain_messages.txt new file mode 100644 index 00000000..c9602d48 --- /dev/null +++ b/packages/langchain-messages/api-report/launchdarkly_ai_langchain_messages.txt @@ -0,0 +1,3 @@ +# Public API of launchdarkly_ai_langchain_messages. Generated by `make api-report`; do not edit. +create_langchain_messages_handler(llm: Any = None, *, capture_content: bool | None = None) -> ProviderHandler +langchain_messages(config_key: str, user_input: str, context: LDContext, llm: Any = None, **kwargs: Any) -> Any diff --git a/packages/openai-agents/api-report/launchdarkly_ai_openai_agents.txt b/packages/openai-agents/api-report/launchdarkly_ai_openai_agents.txt new file mode 100644 index 00000000..6b977819 --- /dev/null +++ b/packages/openai-agents/api-report/launchdarkly_ai_openai_agents.txt @@ -0,0 +1,6 @@ +# Public API of launchdarkly_ai_openai_agents. Generated by `make api-report`; do not edit. +build_output_type(output_format: dict[str, Any] | None) -> dict[str, Any] | None +create_openai_agent_handler(*, capture_content: bool | None = None) -> ProviderHandler +openai_agents(config_key: str, user_input: str, context: LDContext, **kwargs: Any) -> Any +openai_graph(key: str, **options: Any) -> Any +to_openai_agents(def_promise: Any, opts: dict[str, Any] | None = None) -> Any diff --git a/packages/openai-messages/api-report/launchdarkly_ai_openai_messages.txt b/packages/openai-messages/api-report/launchdarkly_ai_openai_messages.txt new file mode 100644 index 00000000..23e9598e --- /dev/null +++ b/packages/openai-messages/api-report/launchdarkly_ai_openai_messages.txt @@ -0,0 +1,3 @@ +# Public API of launchdarkly_ai_openai_messages. Generated by `make api-report`; do not edit. +create_openai_messages_handler(*, capture_content: bool | None = None) -> ProviderHandler +openai_messages(config_key: str, user_input: str, context: LDContext, **kwargs: Any) -> Any diff --git a/packages/opencode-agents/api-report/launchdarkly_ai_opencode_agents.txt b/packages/opencode-agents/api-report/launchdarkly_ai_opencode_agents.txt new file mode 100644 index 00000000..c3e6ce4c --- /dev/null +++ b/packages/opencode-agents/api-report/launchdarkly_ai_opencode_agents.txt @@ -0,0 +1,4 @@ +# Public API of launchdarkly_ai_opencode_agents. Generated by `make api-report`; do not edit. +create_opencode_handler(*, base_url: str, capture_content: bool | None = None, model: ModelResolver | None = None, agent: str | None = None, client: httpx.AsyncClient | None = None) -> ProviderHandler +opencode_agents(config_key: str, user_input: str, context: dict[str, Any], **kwargs: Any) -> Any +OpenCodeSessionError(Exception) [class] diff --git a/pyproject.toml b/pyproject.toml index 0dae9678..cad81ab9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,7 @@ dev = [ "pre-commit>=4.6.0", "brotlicffi>=1.0", "langchain-aws>=1", + "griffelib>=2.1.0", ] [tool.pytest.ini_options] diff --git a/scripts/api_report.py b/scripts/api_report.py new file mode 100644 index 00000000..ca1f2f9b --- /dev/null +++ b/scripts/api_report.py @@ -0,0 +1,184 @@ +"""Generate or check the API report of every public module. + +A report records each name in a module's ``__all__`` with its full signature, one +unindented line per name; a class's public members follow on lines indented by two +spaces. Reports live at ``packages//api-report/.txt`` and are the +package's public API contract: any change to a public name or signature shows up as a +diff in a committed file. + + uv run python scripts/api_report.py # write every report + uv run python scripts/api_report.py --check # fail if any report is stale +""" + +from __future__ import annotations + +import difflib +import sys +from pathlib import Path +from typing import Any + +import griffe + +ROOT = Path(__file__).resolve().parent.parent +# Re-exports the core package unchanged, so the core report covers it. +SKIP = {"launchdarkly_ai_python"} +HEADER = "# Public API of {module}. Generated by `make api-report`; do not edit.\n" + + +def public_modules(root: Path) -> dict[str, Path]: + """Map each reported module to its package directory: each package root, and + each module of a package's ``experimental`` subpackage.""" + found: dict[str, Path] = {} + for init in sorted(root.glob("packages/*/src/*/__init__.py")): + package_dir, name = init.parents[2], init.parent.name + if name in SKIP: + continue + found[name] = package_dir + experimental = init.parent / "experimental" + for sub in sorted(experimental.glob("*.py")) if experimental.is_dir() else []: + if not sub.name.startswith("_"): + found[f"{name}.experimental.{sub.stem}"] = package_dir + return found + + +def resolve(loader: griffe.GriffeLoader, module: Any, name: str) -> Any: + """Return what ``from module import name`` gives an application. + + An imported name wins over a submodule of the same name, as it does at run + time (``from .graph import graph`` binds the function, not the module). + """ + obj = module.members.get(name) + path = module.imports.get(name) + if path: + try: + obj = loader.modules_collection.get_member(path) + except KeyError: # imported from another distribution; keep the alias + pass + if obj is None: + raise SystemExit(f"{module.path}: {name} is in __all__ but not defined") + while obj.is_alias: + try: + obj = obj.target + except griffe.AliasResolutionError: + return obj + return obj + + +# Labels that change how a caller uses a name; decorators are shown separately. +SHOWN_LABELS = {"abstractmethod", "async", "classmethod", "staticmethod"} + + +# Decorators already shown as a label, or as the [property] tag. +LABEL_DECORATORS = {"abstractmethod", "classmethod", "property", "staticmethod"} + + +def tags(obj: Any, *extra: str) -> str: + labels = [*extra, *sorted(obj.labels & SHOWN_LABELS)] + decorators = [ + f"@{d.value}" + for d in getattr(obj, "decorators", []) + if str(d.value).split(".")[-1] not in LABEL_DECORATORS | {"setter", "deleter"} + ] + shown = labels + decorators + return f" [{', '.join(shown)}]" if shown else "" + + +def call(obj: Any, name: str) -> str: + """``name(params) -> returns``, under the name the caller uses.""" + signature = obj.signature() + return ( + name + signature[len(obj.name) :] + if signature.startswith(obj.name) + else signature + ) + + +def describe(obj: Any, name: str, indent: str = "") -> list[str]: + if obj.is_alias: # an unresolved re-export from another distribution + return [f"{indent}{name} = {obj.target_path} [re-export]"] + if obj.is_module: + return [f"{indent}{name} [module]"] + if obj.is_function: + if "property" in obj.labels: + return [f"{indent}{name}: {obj.returns}{tags(obj, 'property')}"] + return [f"{indent}{call(obj, name)}{tags(obj)}"] + if obj.is_type_alias: + return [f"{indent}{name} = {obj.value} [type]"] + if obj.is_attribute: + annotation = f": {obj.annotation}" if obj.annotation is not None else "" + if "property" in obj.labels: + return [f"{indent}{name}{annotation}{tags(obj, 'property')}"] + value = f" = {obj.value}" if obj.value is not None else "" + return [f"{indent}{name}{annotation}{value}{tags(obj)}"] + # A class: its bases, then its constructor and public members. + bases = f"({', '.join(str(b) for b in obj.bases)})" if obj.bases else "" + lines = [f"{indent}{name}{bases}{tags(obj, 'class')}"] + if "__init__" in obj.members: + lines.append(f"{indent} {call(obj, '__init__')}") + members = sorted( + (m for m in obj.members.values() if not m.name.startswith("_")), + key=lambda m: m.name.lower(), + ) + for member in members: + lines += describe(member, member.name, indent + " ") + return lines + + +def report(loader: griffe.GriffeLoader, module_path: str) -> str: + module = loader.modules_collection.get_member(module_path) + if not module.exports: + raise SystemExit(f"{module_path}: no __all__") + lines = [HEADER.format(module=module_path).rstrip("\n")] + for name in sorted(module.exports, key=lambda n: (n.lower(), n)): + lines += describe(resolve(loader, module, str(name)), str(name)) + return "\n".join(lines) + "\n" + + +def reports(root: Path = ROOT) -> dict[Path, str]: + """Return {report path: contents} for every public module under ``root``.""" + modules = public_modules(root) + loader = griffe.GriffeLoader( + search_paths=sorted(str(p) for p in root.glob("packages/*/src")) + ) + for top in sorted({m.split(".")[0] for m in modules}): + loader.load(top) + loader.resolve_aliases(external=False, implicit=False) + return { + package / "api-report" / f"{module}.txt": report(loader, module) + for module, package in modules.items() + } + + +def main(argv: list[str]) -> int: + expected = reports() + if argv != ["--check"]: + for path, text in expected.items(): + path.parent.mkdir(exist_ok=True) + path.write_text(text) + print(f"wrote {len(expected)} API reports") + return 0 + stale = [] + for path in sorted(set(expected) | set(ROOT.glob("packages/*/api-report/*.txt"))): + current = path.read_text() if path.exists() else "" + if current != expected.get(path, ""): + stale.append(path) + rel = str(path.relative_to(ROOT)) + sys.stdout.writelines( + difflib.unified_diff( + current.splitlines(keepends=True), + expected.get(path, "").splitlines(keepends=True), + f"{rel} (committed)", + f"{rel} (from the code)", + ) + ) + if stale: + print( + f"\n{len(stale)} API report(s) out of date: run `make api-report` and commit the result." + ) + return 1 + print(f"{len(expected)} API reports are current") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/tests/test_api_report.py b/tests/test_api_report.py new file mode 100644 index 00000000..d62d30db --- /dev/null +++ b/tests/test_api_report.py @@ -0,0 +1,141 @@ +"""The committed API reports are current, and the generator records what callers see.""" + +from __future__ import annotations + +import textwrap +from pathlib import Path + +import pytest + +from scripts import api_report + + +def test_committed_reports_are_current(capsys: pytest.CaptureFixture[str]) -> None: + # A stale report means a public name or signature changed without its report. + assert api_report.main(["--check"]) == 0, capsys.readouterr().out + + +def write(path: Path, source: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(textwrap.dedent(source)) + + +@pytest.fixture +def workspace(tmp_path: Path) -> Path: + src = tmp_path / "packages/demo/src/demo_sdk" + write( + src / "__init__.py", + """ + from json import JSONDecoder + from demo_sdk.run import run + from demo_sdk.types import Options, Result, Usage, make, LIMIT + + __all__ = ["JSONDecoder", "LIMIT", "Options", "Result", "Usage", "make", "run"] + """, + ) + write( + src / "run.py", + """ + async def run(key: str, *, retries: int = 3, **extra: object) -> str: + return key + """, + ) + write( + src / "types.py", + """ + from dataclasses import dataclass + + LIMIT: int = 10 + Options = dict[str, object] + + @dataclass(frozen=True) + class Usage: + total: int = 0 + _secret: int = 0 + + class Result: + def __init__(self, value: str) -> None: + self.value = value + + @property + def size(self) -> int: + return len(self.value) + + @classmethod + def empty(cls) -> "Result": + return cls("") + + def _hidden(self) -> None: ... + + def make(value: str, /, *items: int) -> Result: + return Result(value) + """, + ) + write(src / "experimental/__init__.py", "") + write( + src / "experimental/beta.py", + """ + def try_it() -> None: ... + + __all__ = ["try_it"] + """, + ) + write(tmp_path / "packages/umbrella/src/launchdarkly_ai_python/__init__.py", "") + return tmp_path + + +def test_report_contents(workspace: Path) -> None: + reports = api_report.reports(workspace) + package = workspace / "packages/demo" + assert set(reports) == { + package / "api-report/demo_sdk.txt", + package / "api-report/demo_sdk.experimental.beta.txt", + } + assert reports[package / "api-report/demo_sdk.txt"] == textwrap.dedent( + """\ + # Public API of demo_sdk. Generated by `make api-report`; do not edit. + JSONDecoder = json.JSONDecoder [re-export] + LIMIT: int = 10 + make(value: str, /, *items: int) -> Result + Options = dict[str, object] + Result [class] + __init__(value: str) + empty() -> Result [classmethod] + size: int [property] + value = value + run(key: str, *, retries: int = 3, **extra: object) -> str [async] + Usage [class, @dataclass(frozen=True)] + __init__(total: int = 0, _secret: int = 0) + total: int = 0 + """ + ) + assert reports[package / "api-report/demo_sdk.experimental.beta.txt"].endswith( + "try_it() -> None\n" + ) + + +def test_an_imported_name_wins_over_a_submodule(workspace: Path) -> None: + # `from demo_sdk.run import run` binds the function, not the `run` module. + text = api_report.reports(workspace)[ + workspace / "packages/demo/api-report/demo_sdk.txt" + ] + assert "run(key: str" in text + assert "run [module]" not in text + + +def test_a_module_without_all_fails(workspace: Path) -> None: + write( + workspace / "packages/demo/src/demo_sdk/experimental/beta.py", + "def try_it() -> None: ...\n", + ) + with pytest.raises(SystemExit, match="no __all__"): + api_report.reports(workspace) + + +def test_an_undefined_name_in_all_fails(workspace: Path) -> None: + write( + workspace / "packages/demo/src/demo_sdk/experimental/beta.py", + '__all__ = ["missing"]\n', + ) + with pytest.raises(SystemExit, match="missing is in __all__ but not defined"): + api_report.reports(workspace) diff --git a/uv.lock b/uv.lock index b7c145b0..7545a687 100644 --- a/uv.lock +++ b/uv.lock @@ -28,6 +28,7 @@ members = [ dev = [ { name = "brotlicffi", specifier = ">=1.0" }, { name = "claude-agent-sdk", specifier = ">=0.2.110" }, + { name = "griffelib", specifier = ">=2.1.0" }, { name = "langchain-anthropic", specifier = ">=1.4.8" }, { name = "langchain-aws", specifier = ">=1" }, { name = "langchain-openai", specifier = ">=0.3" }, From 837c24aa6cf73afd126fdc590bd47062f5f9c343 Mon Sep 17 00:00:00 2001 From: Jeff Dupont Date: Tue, 20 Oct 2026 15:33:25 -0700 Subject: [PATCH 2/6] [AIC-3500] chore(AIC-3500): fail CI on an undeclared breaking change to the public API scripts/api_breaking.py compares the names in each public module's __all__ with another git ref, using griffe: - `pr`: compares with the PR's base. A breaking change passes only if the PR title declares it with `!` (so release-please cuts a major version, or a minor one before 1.0.0), or if an owner applied the api-compatible label. - `release`: on the release-please PR, compares each package whose version changes with its last release tag, and fails on a breaking change unless the new version is a major one (a minor one before 1.0.0). Experimental modules are skipped. griffe reports removed names, parameter changes, changed defaults and changed attribute values; the script adds names dropped from __all__ and changed return types, which griffe does not flag. The API compatibility workflow runs it on every PR, and re-runs it when the title or labels change. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/api-compatibility.yml | 39 ++++ scripts/api_breaking.py | 241 ++++++++++++++++++++++++ scripts/api_report.py | 8 +- tests/test_api_breaking.py | 203 ++++++++++++++++++++ 4 files changed, 488 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/api-compatibility.yml create mode 100644 scripts/api_breaking.py create mode 100644 tests/test_api_breaking.py diff --git a/.github/workflows/api-compatibility.yml b/.github/workflows/api-compatibility.yml new file mode 100644 index 00000000..dd082c54 --- /dev/null +++ b/.github/workflows/api-compatibility.yml @@ -0,0 +1,39 @@ +name: API compatibility + +# A breaking change to a public name or signature must be declared with `!` in the PR +# title, and a release's versions must match the changes since each package's last +# release. See scripts/api_breaking.py. + +on: + pull_request: + # edited / labeled / unlabeled: a new title or the api-compatible label re-runs the check. + types: [opened, synchronize, reopened, edited, labeled, unlabeled] + +permissions: + contents: read + +jobs: + api-compatibility: + name: API compatibility + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # the base commit and every release tag + - uses: astral-sh/setup-uv@v5 + with: + python-version: "3.12" + - run: uv lock + - run: uv sync --all-packages + - name: A breaking change is declared + if: github.head_ref != 'release-please--branches--main' + env: + BASE: ${{ github.event.pull_request.base.sha }} + TITLE: ${{ github.event.pull_request.title }} + LABELS: ${{ join(github.event.pull_request.labels.*.name, ',') }} + run: uv run python -m scripts.api_breaking pr --base "$BASE" --title "$TITLE" --labels "$LABELS" + - name: The release versions match their changes + if: github.head_ref == 'release-please--branches--main' + env: + BASE: ${{ github.event.pull_request.base.sha }} + run: uv run python -m scripts.api_breaking release --base "$BASE" diff --git a/scripts/api_breaking.py b/scripts/api_breaking.py new file mode 100644 index 00000000..a7ea62a0 --- /dev/null +++ b/scripts/api_breaking.py @@ -0,0 +1,241 @@ +"""Fail on an undeclared breaking change to the public API. + + uv run python -m scripts.api_breaking pr --base --title [--labels a,b] + uv run python -m scripts.api_breaking release --base + +``pr`` compares each public module with ``--base``. A breaking change passes only if +the PR title declares it with ``!`` (``feat!: ...``, ``fix(client)!: ...``), which makes +release-please cut a major version (a minor one before 1.0.0), or if an owner applied +the ``api-compatible`` label because they judge the flagged change compatible. + +``release`` runs on the release PR. For each package whose version changes, it +compares the package's public modules with its last release tag, and fails on a +breaking change unless the new version is a major one (a minor one before 1.0.0). + +Only names in ``__all__`` count. Experimental modules are skipped: an experimental +feature may change in a minor release. +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import subprocess +import sys +from pathlib import Path +from typing import Any + +import griffe + +from scripts.api_report import ROOT, public_modules, resolve + +BREAKING_TITLE = re.compile(r"^[a-z]+(\([^)]*\))?!:") +COMPATIBLE_LABEL = "api-compatible" + + +def stable_modules(root: Path) -> dict[str, Path]: + return {m: p for m, p in public_modules(root).items() if ".experimental." not in m} + + +def load(root: Path, ref: str | None, tops: set[str]) -> griffe.ModulesCollection: + """Load the top-level packages ``tops`` from the working tree, or from ``ref``.""" + loader = griffe.GriffeLoader( + search_paths=sorted(str(p) for p in root.glob("packages/*/src")) + ) + relative = sorted(str(p.relative_to(root)) for p in root.glob("packages/*/src")) + for top in sorted(tops): + if ref is None: + loader.load(top) + continue + try: + griffe.load_git( + top, + ref=ref, + repo=root, + search_paths=relative, + modules_collection=loader.modules_collection, + ) + except (ImportError, griffe.GitError): # the package did not exist yet + pass + loader.resolve_aliases(external=False, implicit=False) + return loader.modules_collection + + +def compare( + old: griffe.ModulesCollection, new: griffe.ModulesCollection, modules: list[str] +) -> list[str]: + """Explain every breaking change to the ``__all__`` names of ``modules``. + + griffe compares the two modules' public members: with ``__all__`` defined, only + its names, following re-exports to where they are defined. A name dropped from + ``__all__`` while its object still exists is breaking too, and griffe does not + see it, so it is checked here. + """ + style = ( + griffe.ExplanationStyle.GITHUB + if os.environ.get("GITHUB_ACTIONS") + else griffe.ExplanationStyle.MARKDOWN + ) + found: list[str] = [] + for module in modules: + try: + old_module = old.get_member(module) + except KeyError: # a new module breaks nothing + continue + try: + new_module = new.get_member(module) + except KeyError: + found.append(f"{module}: the module was removed") + continue + dropped = sorted( + set(map(str, old_module.exports or [])) + - set(map(str, new_module.exports or [])) + ) + found += [f"{module}.{name}: removed from __all__" for name in dropped] + found += [ + b.explain(style) + for b in griffe.find_breaking_changes(old_module, new_module) + if not ( + b.kind is griffe.BreakageKind.OBJECT_REMOVED and b.obj.name in dropped + ) + and ".experimental." not in f"{b.obj.path}." + ] + shared = sorted( + set(map(str, old_module.exports or [])) + & set(map(str, new_module.exports or [])) + ) + for name in shared: + found += returns_changed( + f"{module}.{name}", + resolve(old, old_module, name), + resolve(new, new_module, name), + ) + return found + + +def returns_changed(path: str, old: Any, new: Any) -> list[str]: + """griffe does not flag a changed return type, which callers may not be able to handle.""" + pairs = [(path, old, new)] + if old.is_class and new.is_class: + pairs += [ + (f"{path}.{name}", member, new.members[name]) + for name, member in old.members.items() + if not name.startswith("_") and name in new.members + ] + return [ + f"{where}: return type changed from {o.returns} to {n.returns}" + for where, o, n in pairs + if not o.is_alias + and not n.is_alias + and o.is_function + and n.is_function + and str(o.returns) != str(n.returns) + ] + + +def tops_of(modules: list[str]) -> set[str]: + return {m.split(".")[0] for m in modules} + + +def check_pr(root: Path, base: str, title: str, labels: list[str]) -> int: + modules = sorted(stable_modules(root)) + found = compare( + load(root, base, tops_of(modules)), load(root, None, tops_of(modules)), modules + ) + if not found: + print("No breaking changes to the public API.") + return 0 + print("\n".join(found)) + if BREAKING_TITLE.match(title): + print( + "\nDeclared breaking with `!` in the PR title: release-please will cut a major version " + "(a minor one before 1.0.0)." + ) + return 0 + if COMPATIBLE_LABEL in labels: + print( + f"\nAn owner judged these changes compatible (`{COMPATIBLE_LABEL}` label)." + ) + return 0 + print( + f"\n{len(found)} breaking change(s) to the public API. Declare a breaking change with `!` " + "in the PR title (`feat!: ...`, `fix(scope)!: ...`). If an owner judges the change " + f"compatible, they apply the `{COMPATIBLE_LABEL}` label instead." + ) + return 1 + + +def version(text: str) -> tuple[int, int, int]: + major, minor, patch = (int(n) for n in re.findall(r"\d+", text)[:3]) + return major, minor, patch + + +def allows_breaking(old: str, new: str) -> bool: + """True if ``old`` → ``new`` is a major bump, or a minor one before 1.0.0.""" + (o_major, o_minor, _), (n_major, n_minor, _) = version(old), version(new) + return n_major > o_major or (o_major == n_major == 0 and n_minor > o_minor) + + +def git(root: Path, *args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run(["git", *args], cwd=root, capture_output=True, text=True) + + +def check_release(root: Path, base: str) -> int: + config = json.loads((root / "release-please-config.json").read_text())["packages"] + new_versions = json.loads((root / ".release-please-manifest.json").read_text()) + old_versions = json.loads( + git(root, "show", f"{base}:.release-please-manifest.json").stdout + ) + by_package: dict[str, list[str]] = {} + for module, package_dir in stable_modules(root).items(): + by_package.setdefault(str(package_dir.relative_to(root)), []).append(module) + failed = False + for package, new in sorted(new_versions.items()): + old = old_versions.get(package) + modules = sorted(by_package.get(package, [])) + if old is None or old == new or not modules: + continue + tag = f"{config[package]['component']}-{old}" + if git(root, "rev-parse", "--verify", "--quiet", f"refs/tags/{tag}").returncode: + print( + f"{package}: no release tag {tag}, so there is nothing to compare with" + ) + continue + found = compare( + load(root, tag, tops_of(modules)), + load(root, None, tops_of(modules)), + modules, + ) + if not found or allows_breaking(old, new): + print(f"{package}: {old} -> {new} matches its changes since {tag}") + continue + failed = True + print("\n".join(found)) + print( + f"{package}: {old} -> {new}, but the changes above since {tag} are breaking. " + "It needs a major version (a minor one before 1.0.0): a merged PR that made one " + "of them was not declared with `!` in its title." + ) + return 1 if failed else 0 + + +def main(argv: list[str]) -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + sub = parser.add_subparsers(dest="command", required=True) + pr = sub.add_parser("pr") + pr.add_argument("--base", required=True) + pr.add_argument("--title", required=True) + pr.add_argument("--labels", default="") + release = sub.add_parser("release") + release.add_argument("--base", required=True) + args = parser.parse_args(argv) + if args.command == "pr": + labels = [label.strip() for label in args.labels.split(",") if label.strip()] + return check_pr(ROOT, args.base, args.title, labels) + return check_release(ROOT, args.base) + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/scripts/api_report.py b/scripts/api_report.py index ca1f2f9b..45c4386b 100644 --- a/scripts/api_report.py +++ b/scripts/api_report.py @@ -41,7 +41,7 @@ def public_modules(root: Path) -> dict[str, Path]: return found -def resolve(loader: griffe.GriffeLoader, module: Any, name: str) -> Any: +def resolve(collection: griffe.ModulesCollection, module: Any, name: str) -> Any: """Return what ``from module import name`` gives an application. An imported name wins over a submodule of the same name, as it does at run @@ -51,7 +51,7 @@ def resolve(loader: griffe.GriffeLoader, module: Any, name: str) -> Any: path = module.imports.get(name) if path: try: - obj = loader.modules_collection.get_member(path) + obj = collection.get_member(path) except KeyError: # imported from another distribution; keep the alias pass if obj is None: @@ -130,7 +130,9 @@ def report(loader: griffe.GriffeLoader, module_path: str) -> str: raise SystemExit(f"{module_path}: no __all__") lines = [HEADER.format(module=module_path).rstrip("\n")] for name in sorted(module.exports, key=lambda n: (n.lower(), n)): - lines += describe(resolve(loader, module, str(name)), str(name)) + lines += describe( + resolve(loader.modules_collection, module, str(name)), str(name) + ) return "\n".join(lines) + "\n" diff --git a/tests/test_api_breaking.py b/tests/test_api_breaking.py new file mode 100644 index 00000000..bb910065 --- /dev/null +++ b/tests/test_api_breaking.py @@ -0,0 +1,203 @@ +"""Breaking changes to the public API must be declared, and releases must match them.""" + +from __future__ import annotations + +import json +import subprocess +import textwrap +from pathlib import Path + +import pytest + +from scripts import api_breaking + +V1 = """ +from demo_sdk.run import run + +def make(value: str) -> str: + return value + +__all__ = ["make", "run"] +""" +RUN = "def run(key: str) -> str:\n return key\n" +BETA = '__all__ = ["try_it"]\n\ndef try_it() -> None: ...\n' + + +def git(repo: Path, *args: str) -> None: + subprocess.run( + ["git", *args], + cwd=repo, + check=True, + capture_output=True, + ) + + +def write(path: Path, text: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(textwrap.dedent(text)) + + +@pytest.fixture +def repo(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + """A one-package repository whose first commit is released as ``demo-sdk-1.2.0``.""" + monkeypatch.delenv("GITHUB_ACTIONS", raising=False) + # Keep a developer's global git config (signing, hooks) out of these repositories, + # including the git that griffe runs to read an old ref. + gitconfig = tmp_path.parent / f"{tmp_path.name}.gitconfig" + gitconfig.write_text("[user]\n\tname = test\n\temail = test@example.com\n") + monkeypatch.setenv("GIT_CONFIG_GLOBAL", str(gitconfig)) + monkeypatch.setenv("GIT_CONFIG_NOSYSTEM", "1") + src = tmp_path / "packages/demo/src/demo_sdk" + write(src / "__init__.py", V1) + write(src / "run.py", RUN) + write(src / "experimental/__init__.py", "") + write(src / "experimental/beta.py", BETA) + write( + tmp_path / "release-please-config.json", + json.dumps({"packages": {"packages/demo": {"component": "demo-sdk"}}}), + ) + write( + tmp_path / ".release-please-manifest.json", + json.dumps({"packages/demo": "1.2.0"}), + ) + git(tmp_path, "init", "-q") + git(tmp_path, "add", "-A") + git(tmp_path, "commit", "-q", "-m", "v1") + git(tmp_path, "tag", "demo-sdk-1.2.0") + return tmp_path + + +def change(repo: Path, relative: str, text: str) -> None: + write(repo / "packages/demo/src/demo_sdk" / relative, text) + git(repo, "commit", "-q", "-am", "change") + + +def breakages(repo: Path) -> list[str]: + modules = sorted(api_breaking.stable_modules(repo)) + tops = api_breaking.tops_of(modules) + return api_breaking.compare( + api_breaking.load(repo, "demo-sdk-1.2.0", tops), + api_breaking.load(repo, None, tops), + modules, + ) + + +def test_no_change_breaks_nothing(repo: Path) -> None: + assert breakages(repo) == [] + + +@pytest.mark.parametrize( + ("source", "expected"), + [ + ("def run(key: str, strict: bool) -> str: ...\n", "added as required"), + ("def run(name: str) -> str: ...\n", "removed"), + ("def run(key: str) -> int: ...\n", "return type changed from str to int"), + ], +) +def test_a_breaking_signature_change_is_found( + repo: Path, source: str, expected: str +) -> None: + change(repo, "run.py", source) + found = breakages(repo) + assert len(found) >= 1 + assert any(expected.lower() in line.lower() for line in found), found + + +def test_an_added_optional_parameter_or_name_breaks_nothing(repo: Path) -> None: + change(repo, "run.py", "def run(key: str, strict: bool = False) -> str: ...\n") + change( + repo, + "__init__.py", + V1.replace('"make", "run"', '"extra", "make", "run"') + "extra = 1\n", + ) + assert breakages(repo) == [] + + +def test_a_name_removed_from_all_is_breaking(repo: Path) -> None: + change(repo, "__init__.py", V1.replace('"make", "run"', '"run"')) + assert breakages(repo) == ["demo_sdk.make: removed from __all__"] + + +def test_an_experimental_module_may_break(repo: Path) -> None: + change( + repo, + "experimental/beta.py", + '__all__ = ["try_it"]\n\ndef try_it(x: int) -> None: ...\n', + ) + assert breakages(repo) == [] + + +@pytest.mark.parametrize( + ("title", "labels", "code"), + [ + ("feat: change run", [], 1), + ("feat!: change run", [], 0), + ("fix(client)!: change run", [], 0), + ("feat: !important", [], 1), + ("feat: change run", ["api-compatible"], 0), + ], +) +def test_a_breaking_pr_must_declare_it( + repo: Path, title: str, labels: list[str], code: int +) -> None: + change(repo, "run.py", "def run(key: str, strict: bool) -> str: ...\n") + assert api_breaking.check_pr(repo, "demo-sdk-1.2.0", title, labels) == code + + +def test_a_pr_without_breaking_changes_passes(repo: Path) -> None: + assert api_breaking.check_pr(repo, "demo-sdk-1.2.0", "feat: nothing", []) == 0 + + +def release(repo: Path, released: str, proposed: str, breaking: bool) -> int: + """Release ``released`` as tagged, then propose ``proposed`` the way a release PR does.""" + git(repo, "tag", "-d", "demo-sdk-1.2.0") + write( + repo / ".release-please-manifest.json", json.dumps({"packages/demo": released}) + ) + git(repo, "commit", "-q", "--allow-empty", "-am", "released") + git(repo, "tag", f"demo-sdk-{released}") + base = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=repo, + capture_output=True, + text=True, + check=True, + ).stdout.strip() + if breaking: + change(repo, "run.py", "def run(key: str, strict: bool) -> str: ...\n") + write( + repo / ".release-please-manifest.json", json.dumps({"packages/demo": proposed}) + ) + git(repo, "commit", "-q", "-am", "release PR") + return api_breaking.check_release(repo, base) + + +@pytest.mark.parametrize( + ("released", "proposed", "breaking", "code"), + [ + ("1.2.0", "1.3.0", True, 1), + ("1.2.0", "1.2.1", True, 1), + ("1.2.0", "2.0.0", True, 0), + ("1.2.0", "1.3.0", False, 0), + ("0.3.0", "0.4.0", True, 0), + ("0.3.0", "0.3.1", True, 1), + ("0.9.0", "1.0.0", True, 0), + ], +) +def test_a_release_version_matches_its_changes( + repo: Path, released: str, proposed: str, breaking: bool, code: int +) -> None: + assert release(repo, released, proposed, breaking) == code + + +def test_a_package_with_no_release_tag_is_skipped( + repo: Path, capsys: pytest.CaptureFixture[str] +) -> None: + base = "HEAD" + git(repo, "tag", "-d", "demo-sdk-1.2.0") + write( + repo / ".release-please-manifest.json", json.dumps({"packages/demo": "1.3.0"}) + ) + git(repo, "commit", "-q", "-am", "release PR") + assert api_breaking.check_release(repo, f"{base}~1") == 0 + assert "no release tag demo-sdk-1.2.0" in capsys.readouterr().out From b7acfb08cdd60a7d335eb55da1811f5e37185528 Mon Sep 17 00:00:00 2001 From: Jeff Dupont Date: Tue, 20 Oct 2026 15:33:26 -0700 Subject: [PATCH 3/6] [AIC-3500] chore(AIC-3500): let api-compatible pass the release check too An owner can now judge a flagged change compatible on the release PR, as on any other PR. Without it, a compatible change flagged on the release PR (the capture_content default on #166) could only pass by bumping the version. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/api-compatibility.yml | 3 ++- scripts/api_breaking.py | 22 +++++++++++++++++----- tests/test_api_breaking.py | 19 +++++++++++++++++-- 3 files changed, 36 insertions(+), 8 deletions(-) diff --git a/.github/workflows/api-compatibility.yml b/.github/workflows/api-compatibility.yml index dd082c54..2308d125 100644 --- a/.github/workflows/api-compatibility.yml +++ b/.github/workflows/api-compatibility.yml @@ -36,4 +36,5 @@ jobs: if: github.head_ref == 'release-please--branches--main' env: BASE: ${{ github.event.pull_request.base.sha }} - run: uv run python -m scripts.api_breaking release --base "$BASE" + LABELS: ${{ join(github.event.pull_request.labels.*.name, ',') }} + run: uv run python -m scripts.api_breaking release --base "$BASE" --labels "$LABELS" diff --git a/scripts/api_breaking.py b/scripts/api_breaking.py index a7ea62a0..c3f6947b 100644 --- a/scripts/api_breaking.py +++ b/scripts/api_breaking.py @@ -1,7 +1,7 @@ """Fail on an undeclared breaking change to the public API. uv run python -m scripts.api_breaking pr --base --title [--labels a,b] - uv run python -m scripts.api_breaking release --base + uv run python -m scripts.api_breaking release --base [--labels a,b] ``pr`` compares each public module with ``--base``. A breaking change passes only if the PR title declares it with ``!`` (``feat!: ...``, ``fix(client)!: ...``), which makes @@ -10,7 +10,8 @@ ``release`` runs on the release PR. For each package whose version changes, it compares the package's public modules with its last release tag, and fails on a -breaking change unless the new version is a major one (a minor one before 1.0.0). +breaking change unless the new version is a major one (a minor one before 1.0.0), or +an owner applied the ``api-compatible`` label to the release PR. Only names in ``__all__`` count. Experimental modules are skipped: an experimental feature may change in a minor release. @@ -182,7 +183,7 @@ def git(root: Path, *args: str) -> subprocess.CompletedProcess[str]: return subprocess.run(["git", *args], cwd=root, capture_output=True, text=True) -def check_release(root: Path, base: str) -> int: +def check_release(root: Path, base: str, labels: list[str] | None = None) -> int: config = json.loads((root / "release-please-config.json").read_text())["packages"] new_versions = json.loads((root / ".release-please-manifest.json").read_text()) old_versions = json.loads( @@ -218,6 +219,16 @@ def check_release(root: Path, base: str) -> int: "It needs a major version (a minor one before 1.0.0): a merged PR that made one " "of them was not declared with `!` in its title." ) + if failed and COMPATIBLE_LABEL in (labels or []): + print( + f"\nAn owner judged these changes compatible (`{COMPATIBLE_LABEL}` label)." + ) + return 0 + if failed: + print( + f"\nIf an owner judges these changes compatible, they apply the `{COMPATIBLE_LABEL}` " + "label to this release PR." + ) return 1 if failed else 0 @@ -230,11 +241,12 @@ def main(argv: list[str]) -> int: pr.add_argument("--labels", default="") release = sub.add_parser("release") release.add_argument("--base", required=True) + release.add_argument("--labels", default="") args = parser.parse_args(argv) + labels = [label.strip() for label in args.labels.split(",") if label.strip()] if args.command == "pr": - labels = [label.strip() for label in args.labels.split(",") if label.strip()] return check_pr(ROOT, args.base, args.title, labels) - return check_release(ROOT, args.base) + return check_release(ROOT, args.base, labels) if __name__ == "__main__": diff --git a/tests/test_api_breaking.py b/tests/test_api_breaking.py index bb910065..c695021b 100644 --- a/tests/test_api_breaking.py +++ b/tests/test_api_breaking.py @@ -148,7 +148,13 @@ def test_a_pr_without_breaking_changes_passes(repo: Path) -> None: assert api_breaking.check_pr(repo, "demo-sdk-1.2.0", "feat: nothing", []) == 0 -def release(repo: Path, released: str, proposed: str, breaking: bool) -> int: +def release( + repo: Path, + released: str, + proposed: str, + breaking: bool, + labels: list[str] | None = None, +) -> int: """Release ``released`` as tagged, then propose ``proposed`` the way a release PR does.""" git(repo, "tag", "-d", "demo-sdk-1.2.0") write( @@ -169,7 +175,7 @@ def release(repo: Path, released: str, proposed: str, breaking: bool) -> int: repo / ".release-please-manifest.json", json.dumps({"packages/demo": proposed}) ) git(repo, "commit", "-q", "-am", "release PR") - return api_breaking.check_release(repo, base) + return api_breaking.check_release(repo, base, labels) @pytest.mark.parametrize( @@ -201,3 +207,12 @@ def test_a_package_with_no_release_tag_is_skipped( git(repo, "commit", "-q", "-am", "release PR") assert api_breaking.check_release(repo, f"{base}~1") == 0 assert "no release tag demo-sdk-1.2.0" in capsys.readouterr().out + + +def test_an_owner_can_judge_a_release_compatible(repo: Path) -> None: + # The label passes an undeclared breaking change on the release PR, as on any other PR. + assert release(repo, "1.2.0", "1.2.1", True, ["api-compatible"]) == 0 + + +def test_another_label_does_not_pass_a_breaking_release(repo: Path) -> None: + assert release(repo, "1.2.0", "1.2.1", True, ["docs"]) == 1 From 0d155e6e95549a638ef36a51407fcc291cbb640d Mon Sep 17 00:00:00 2001 From: Jeff Dupont Date: Thu, 22 Oct 2026 09:19:05 -0700 Subject: [PATCH 4/6] chore(AIC-3500): include public dunders and TypedDict keywords in API reports Co-Authored-By: Claude Opus 5.5 --- ...kly_ai_server.experimental.evaluations.txt | 4 ++- ...chdarkly_ai_server.experimental.skills.txt | 4 +++ .../api-report/launchdarkly_ai_server.txt | 2 ++ scripts/api_report.py | 13 ++++++-- tests/test_api_report.py | 33 +++++++++++++++++++ 5 files changed, 52 insertions(+), 4 deletions(-) diff --git a/packages/client/api-report/launchdarkly_ai_server.experimental.evaluations.txt b/packages/client/api-report/launchdarkly_ai_server.experimental.evaluations.txt index b4abf15e..8c7dde8f 100644 --- a/packages/client/api-report/launchdarkly_ai_server.experimental.evaluations.txt +++ b/packages/client/api-report/launchdarkly_ai_server.experimental.evaluations.txt @@ -36,7 +36,7 @@ EvaluationsModule [class] sdk_key: str | None [property] tools: ToolsClient [property] ui_base_uri: str [property] -GenerationConfig(TypedDict) [class] +GenerationConfig(TypedDict, total=False) [class] instructions: str messages: list[dict[str, Any]] model: str @@ -47,6 +47,7 @@ GenerationConfig(TypedDict) [class] init_evaluations(project_key: str | None = None, api_key: str | None = None, sdk_key: str | None = None, base_uri: str | None = None, ui_base_uri: str | None = None, transport: Transport = urllib_transport) -> EvaluationsModule Judge [class, @dataclass(frozen=True)] __init__(key: str, threshold: float | None = DEFAULT_JUDGE_THRESHOLD, pass_rate_threshold: float | None = None, ground_truth_context: str | None = None) + __post_init__() -> None criterion_type: str [property] ground_truth_context: str | None = None key: str @@ -63,6 +64,7 @@ RunSummary [class, @dataclass] total_rows: int = 0 Scorer [class, @dataclass(frozen=True)] __init__(name: str, fn: ScorerFn, threshold: float | None = 1.0, pass_rate_threshold: float | None = None, success_direction: SuccessDirection = 'higher_is_better') + __post_init__() -> None criterion_type: str [property] fn: ScorerFn name: str diff --git a/packages/client/api-report/launchdarkly_ai_server.experimental.skills.txt b/packages/client/api-report/launchdarkly_ai_server.experimental.skills.txt index 18639cd0..11f3fee8 100644 --- a/packages/client/api-report/launchdarkly_ai_server.experimental.skills.txt +++ b/packages/client/api-report/launchdarkly_ai_server.experimental.skills.txt @@ -2,6 +2,8 @@ all_skills() -> list[Skill] [async] FDv2SkillStore [class] __init__(sdk_key: str, *, base_uri: str | None = None, stream_uri: str | None = None, mode: Mode = 'stream', poll_interval: float = 30.0, read_timeout: float | None = None, initial_backoff: float = 1.0, max_backoff: float = 30.0, _requester: Any = None) + __enter__() -> FDv2SkillStore + __exit__(*_exc: Any) -> None add_listener(kind: str, fn: Callable[[dict[str, Any]], Any]) -> None all_objects(kind: str) -> dict[str, dict[str, Any]] close(timeout: float = 5.0) -> None @@ -64,6 +66,8 @@ SkillStore(Protocol) [class] get_object(kind: str, key: str, version: int | None = None) -> dict[str, Any] | None SkillWatcher [class] __init__(request: Sequence[Skill | SkillReference | str] | str, root: str | os.PathLike[str], store: Any, *, prune: bool, timeout: float, on_unavailable: OnUnavailable, debounce: float, on_reconcile: Callable[[ReconcileReport], Any] | None) + __enter__() -> SkillWatcher + __exit__(*_exc: Any) -> None close(timeout: float = 15.0) -> None notify(_raw: Any = None) -> None reconciles: int [property] diff --git a/packages/client/api-report/launchdarkly_ai_server.txt b/packages/client/api-report/launchdarkly_ai_server.txt index ecc9ea06..fd1861f4 100644 --- a/packages/client/api-report/launchdarkly_ai_server.txt +++ b/packages/client/api-report/launchdarkly_ai_server.txt @@ -148,6 +148,7 @@ ProviderGraphResponse [class, @dataclass] usage: UsageDict ProviderHandler [class] __init__(fn: _HandlerFn, provides_for: tuple[str, Literal['agent', 'messages']] | None = None, stream_fn: _StreamFn | None = None, capture_content: bool | None = None) + __call__(config: AiConfigRep, user_input: str | None = None, tool_handlers: dict[str, Any] | None = None, variables: dict[str, Any] | None = None, history: list[dict[str, Any]] | None = None) -> HandlerResult [async] capture_content: bool [property] has_stream: bool [property] provides_for: tuple[str, Literal['agent', 'messages']] | None = provides_for @@ -180,6 +181,7 @@ SDK_INFO_CONTEXT: LDContext = {'kind': 'ld_ai', 'key': 'ld-internal-tracking', ' SDK_INFO_EVENT = '$ld:ai:sdk:info' SDKOptions [class, @dataclass(frozen=True)] __init__(capture_content: bool = False) + __post_init__() -> None capture_content: bool = False set_conversation_id_if_absent(span: Any, conversation: str) -> None set_input_content_attributes(span: Any, capture: bool, *, system_instructions: str | None = None, messages: list[SpanMessage] | None = None, tool_definitions: list[ToolDefinitionInput] | None = None) -> None diff --git a/scripts/api_report.py b/scripts/api_report.py index ca1f2f9b..89605c7f 100644 --- a/scripts/api_report.py +++ b/scripts/api_report.py @@ -110,13 +110,20 @@ def describe(obj: Any, name: str, indent: str = "") -> list[str]: return [f"{indent}{name}{annotation}{tags(obj, 'property')}"] value = f" = {obj.value}" if obj.value is not None else "" return [f"{indent}{name}{annotation}{value}{tags(obj)}"] - # A class: its bases, then its constructor and public members. - bases = f"({', '.join(str(b) for b in obj.bases)})" if obj.bases else "" + # A class: its bases and keywords (e.g. ``total=False``), then its constructor and + # public members, dunders included. + args = [*map(str, obj.bases), *(f"{k}={v}" for k, v in obj.keywords.items())] + bases = f"({', '.join(args)})" if args else "" lines = [f"{indent}{name}{bases}{tags(obj, 'class')}"] if "__init__" in obj.members: lines.append(f"{indent} {call(obj, '__init__')}") members = sorted( - (m for m in obj.members.values() if not m.name.startswith("_")), + ( + m + for m in obj.members.values() + if m.name != "__init__" + and (not m.name.startswith("_") or m.name.endswith("__")) + ), key=lambda m: m.name.lower(), ) for member in members: diff --git a/tests/test_api_report.py b/tests/test_api_report.py index d62d30db..800d26e9 100644 --- a/tests/test_api_report.py +++ b/tests/test_api_report.py @@ -139,3 +139,36 @@ def test_an_undefined_name_in_all_fails(workspace: Path) -> None: ) with pytest.raises(SystemExit, match="missing is in __all__ but not defined"): api_report.reports(workspace) + + +def test_dunders_and_class_keywords_are_reported(workspace: Path) -> None: + write( + workspace / "packages/demo/src/demo_sdk/experimental/beta.py", + """ + from typing import TypedDict + + class Config(TypedDict, total=False): + model: str + + class Runner: + def __init__(self) -> None: ... + def __call__(self, key: str) -> int: ... + def _private(self) -> None: ... + + __all__ = ["Config", "Runner"] + """, + ) + text = api_report.reports(workspace)[ + workspace / "packages/demo/api-report/demo_sdk.experimental.beta.txt" + ] + assert text.endswith( + textwrap.dedent( + """\ + Config(TypedDict, total=False) [class] + model: str + Runner [class] + __init__() + __call__(key: str) -> int + """ + ) + ) From a61cf9e7101787860b148243361d4237ad1f43cc Mon Sep 17 00:00:00 2001 From: Jeff Dupont Date: Thu, 22 Oct 2026 09:21:35 -0700 Subject: [PATCH 5/6] chore(AIC-3500): catch removed packages and dunder changes in the breaking check Co-Authored-By: Claude Opus 5.5 --- scripts/api_breaking.py | 25 +++++++++++++++++++------ tests/test_api_breaking.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 6 deletions(-) diff --git a/scripts/api_breaking.py b/scripts/api_breaking.py index c3f6947b..4ad8832e 100644 --- a/scripts/api_breaking.py +++ b/scripts/api_breaking.py @@ -30,7 +30,7 @@ import griffe -from scripts.api_report import ROOT, public_modules, resolve +from scripts.api_report import ROOT, SKIP, public_modules, resolve BREAKING_TITLE = re.compile(r"^[a-z]+(\([^)]*\))?!:") COMPATIBLE_LABEL = "api-compatible" @@ -45,17 +45,22 @@ def load(root: Path, ref: str | None, tops: set[str]) -> griffe.ModulesCollectio loader = griffe.GriffeLoader( search_paths=sorted(str(p) for p in root.glob("packages/*/src")) ) - relative = sorted(str(p.relative_to(root)) for p in root.glob("packages/*/src")) + relative = {str(p.relative_to(root)) for p in root.glob("packages/*/src")} + if ref is not None: # and the ref's own, for a package removed since + relative |= {src for src, _ in package_roots(root, ref)} for top in sorted(tops): if ref is None: - loader.load(top) + try: + loader.load(top) + except ImportError: # the package was removed; compare reports it + pass continue try: griffe.load_git( top, ref=ref, repo=root, - search_paths=relative, + search_paths=sorted(relative), modules_collection=loader.modules_collection, ) except (ImportError, griffe.GitError): # the package did not exist yet @@ -123,7 +128,7 @@ def returns_changed(path: str, old: Any, new: Any) -> list[str]: pairs += [ (f"{path}.{name}", member, new.members[name]) for name, member in old.members.items() - if not name.startswith("_") and name in new.members + if (not name.startswith("_") or name.endswith("__")) and name in new.members ] return [ f"{where}: return type changed from {o.returns} to {n.returns}" @@ -140,8 +145,16 @@ def tops_of(modules: list[str]) -> set[str]: return {m.split(".")[0] for m in modules} +def package_roots(root: Path, ref: str) -> list[tuple[str, str]]: + """(source directory, top-level package) of each package at ``ref``.""" + files = git(root, "ls-tree", "-r", "--name-only", ref, "--", "packages").stdout + return re.findall(r"^(packages/[^/]+/src)/([^/]+)/__init__\.py$", files, re.M) + + def check_pr(root: Path, base: str, title: str, labels: list[str]) -> int: - modules = sorted(stable_modules(root)) + # The base's package roots too, so a package removed or renamed since is compared. + at_base = {top for _, top in package_roots(root, base)} - SKIP + modules = sorted(set(stable_modules(root)) | at_base) found = compare( load(root, base, tops_of(modules)), load(root, None, tops_of(modules)), modules ) diff --git a/tests/test_api_breaking.py b/tests/test_api_breaking.py index c695021b..21dac5a7 100644 --- a/tests/test_api_breaking.py +++ b/tests/test_api_breaking.py @@ -216,3 +216,33 @@ def test_an_owner_can_judge_a_release_compatible(repo: Path) -> None: def test_another_label_does_not_pass_a_breaking_release(repo: Path) -> None: assert release(repo, "1.2.0", "1.2.1", True, ["docs"]) == 1 + + +@pytest.mark.parametrize( + ("title", "labels", "code"), + [("feat: drop demo", [], 1), ("feat: drop demo", ["api-compatible"], 0)], +) +def test_a_removed_package_is_breaking( + repo: Path, + capsys: pytest.CaptureFixture[str], + title: str, + labels: list[str], + code: int, +) -> None: + git(repo, "rm", "-rq", "packages/demo/src") + git(repo, "commit", "-q", "-m", "drop demo") + assert api_breaking.check_pr(repo, "demo-sdk-1.2.0", title, labels) == code + assert "demo_sdk: the module was removed" in capsys.readouterr().out + + +def test_a_dunder_return_type_change_is_breaking( + repo: Path, capsys: pytest.CaptureFixture[str] +) -> None: + runner = "class Runner:\n def __call__(self) -> str: ...\n" + exported = V1.replace('"make", "run"', '"Runner", "make", "run"') + change(repo, "__init__.py", exported + runner) + change(repo, "__init__.py", exported + runner.replace("-> str", "-> int")) + assert api_breaking.check_pr(repo, "HEAD~1", "feat: change Runner", []) == 1 + assert "Runner.__call__: return type changed from str to int" in ( + capsys.readouterr().out + ) From 33dcfde56e9b7fc86c52cf963f61d8051e7a8228 Mon Sep 17 00:00:00 2001 From: Jeff Dupont Date: Thu, 22 Oct 2026 09:45:44 -0700 Subject: [PATCH 6/6] chore(AIC-3500): remove stale API reports when writing them Co-Authored-By: Claude Opus 5.5 --- scripts/api_report.py | 5 ++++- tests/test_api_report.py | 12 ++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/scripts/api_report.py b/scripts/api_report.py index 89605c7f..5251ebb6 100644 --- a/scripts/api_report.py +++ b/scripts/api_report.py @@ -157,11 +157,14 @@ def reports(root: Path = ROOT) -> dict[Path, str]: def main(argv: list[str]) -> int: - expected = reports() + expected = reports(ROOT) if argv != ["--check"]: for path, text in expected.items(): path.parent.mkdir(exist_ok=True) path.write_text(text) + # A removed or renamed module's old report would otherwise stay and fail --check. + for path in set(ROOT.glob("packages/*/api-report/*.txt")) - set(expected): + path.unlink() print(f"wrote {len(expected)} API reports") return 0 stale = [] diff --git a/tests/test_api_report.py b/tests/test_api_report.py index 800d26e9..c8e1b1b6 100644 --- a/tests/test_api_report.py +++ b/tests/test_api_report.py @@ -123,6 +123,18 @@ def test_an_imported_name_wins_over_a_submodule(workspace: Path) -> None: assert "run [module]" not in text +def test_writing_removes_a_stale_report( + workspace: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(api_report, "ROOT", workspace) + stale = workspace / "packages/demo/api-report/demo_sdk.gone.txt" + stale.parent.mkdir() + stale.write_text("old\n") + assert api_report.main([]) == 0 + assert not stale.exists() + assert api_report.main(["--check"]) == 0 + + def test_a_module_without_all_fails(workspace: Path) -> None: write( workspace / "packages/demo/src/demo_sdk/experimental/beta.py",