diff --git a/.changes/core-app-separation.added.md b/.changes/core-app-separation.added.md new file mode 100644 index 00000000..f9b74a59 --- /dev/null +++ b/.changes/core-app-separation.added.md @@ -0,0 +1 @@ +新增可独立构建的 inkcre_core SDK，显式绑定数据库与 Peer 执行上下文；服务保留环境、HTTP、调度和安装装配，旧扩展导入继续兼容。 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a177a5cb..f518cc99 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -63,7 +63,7 @@ jobs: cache: true - name: Install locked dependencies - run: pdm install -G dev --frozen-lockfile + run: pdm install -G app -G dev --frozen-lockfile - name: Enforce release intent env: diff --git a/.github/workflows/extension-documentation-publish.yml b/.github/workflows/extension-documentation-publish.yml index 011fb5f6..9cee2505 100644 --- a/.github/workflows/extension-documentation-publish.yml +++ b/.github/workflows/extension-documentation-publish.yml @@ -75,7 +75,7 @@ jobs: cache: true - name: Install frozen publisher - run: pdm install --no-default --group extension-publisher --frozen-lockfile + run: pdm install --no-self --no-default --group extension-publisher --frozen-lockfile - name: Set up pnpm uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9 @@ -126,7 +126,7 @@ jobs: cache: true - name: Install frozen publisher - run: pdm install --no-default --group extension-publisher --frozen-lockfile + run: pdm install --no-self --no-default --group extension-publisher --frozen-lockfile - name: Restore exact documentation candidates uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 diff --git a/.github/workflows/extension-publish.yml b/.github/workflows/extension-publish.yml index d3fc5edb..1935c164 100644 --- a/.github/workflows/extension-publish.yml +++ b/.github/workflows/extension-publish.yml @@ -112,7 +112,7 @@ jobs: - name: Install frozen publisher if: steps.selection.outputs.selected == 'true' run: >- - pdm install --no-default --group extension-publisher --frozen-lockfile + pdm install --no-self --no-default --group extension-publisher --frozen-lockfile - name: Build and verify native wheel if: steps.selection.outputs.selected == 'true' @@ -203,7 +203,7 @@ jobs: - name: Install frozen publisher if: steps.candidate.outputs.selected == 'true' run: >- - pdm install --no-default --group extension-publisher --frozen-lockfile + pdm install --no-self --no-default --group extension-publisher --frozen-lockfile - name: Prepare exact native Release association if: steps.candidate.outputs.selected == 'true' diff --git a/.github/workflows/preview-deploy.yml b/.github/workflows/preview-deploy.yml index 06685060..3782835f 100644 --- a/.github/workflows/preview-deploy.yml +++ b/.github/workflows/preview-deploy.yml @@ -82,7 +82,7 @@ jobs: - name: Install frozen Extension preview tooling working-directory: source run: >- - pdm install --no-default --group extension-preview --frozen-lockfile + pdm install --no-self --no-default --group extension-preview --frozen-lockfile - name: Build exact-head Extension wheels and sibling Registry facade env: diff --git a/.github/workflows/release-pr.yml b/.github/workflows/release-pr.yml index d2b61d3c..2dc3aceb 100644 --- a/.github/workflows/release-pr.yml +++ b/.github/workflows/release-pr.yml @@ -31,7 +31,7 @@ jobs: cache: true - name: Install locked release tooling - run: pdm install -G dev --frozen-lockfile + run: pdm install -G app -G dev --frozen-lockfile - name: Prepare every pending project run: pdm run release:prepare diff --git a/.github/workflows/self-host-render-neon.yml b/.github/workflows/self-host-render-neon.yml index 8dbb5bef..8d0d71be 100644 --- a/.github/workflows/self-host-render-neon.yml +++ b/.github/workflows/self-host-render-neon.yml @@ -41,7 +41,7 @@ jobs: cache: true - name: Install locked runtime dependencies - run: pdm install --prod --no-editable --frozen-lockfile + run: pdm install -G app --prod --no-editable --frozen-lockfile - name: Resolve the Neon default branch owner coordinates id: neon diff --git a/AGENTS.md b/AGENTS.md index 8350f947..c8b16ebb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -30,10 +30,11 @@ Before choosing a design, identify shared state, business contracts, and each pa ## Repository Map - `run.py`: FastAPI bootstrap -- `app/`: application routes, schemas, and business units +- `inkcre_core/`: independently consumable SDK, models, domain operations, persistence and execution +- `app/`: service routes, configuration, extension/sink composition and legacy import shims - `cli/`: `inkcre-cli` implementation - `extensions/`: built-in first-party extension packages -- `libs/`: shared libraries +- `libs/`, `utils/`: legacy imports; implementations live under `inkcre_core/` - `migrations/`: Alembic revisions and integrity baseline (append-only after deployment) - `tests/`: migration integrity and mature integration/acceptance checks - `docs/30-unit-tdd/`: `impl-py` internal design diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 84fcdd8e..edd2e096 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -15,7 +15,7 @@ bugs that do not cross a security boundary. ```bash git submodule update --init --recursive -pdm install -G dev --frozen-lockfile +pdm install -G app -G dev --frozen-lockfile pdm run pre-commit install pdm run doctor pdm run check diff --git a/Dockerfile b/Dockerfile index 8f209f10..742f479d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,8 +14,9 @@ WORKDIR /app # PDM 2.28 requires the httpx client replaced in unearth 0.19. RUN pip install --no-cache-dir "pdm==${PDM_VERSION}" "unearth==0.18.3" -COPY pyproject.toml pdm.lock README.md ./ -RUN pdm install --prod --no-editable --frozen-lockfile +COPY pyproject.toml pdm.lock README.md LICENSE ./ +COPY inkcre_core/ inkcre_core/ +RUN pdm install -G app --prod --no-editable --frozen-lockfile FROM python:${PYTHON_VERSION}-slim AS runtime diff --git a/README.md b/README.md index d5e32cfd..c4e8315d 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ You do not need to clone this repository. Fork it on GitHub, configure the docum ```bash git submodule update --init --recursive -pdm install -G dev --frozen-lockfile +pdm install -G app -G dev --frozen-lockfile svc dev ensure database --repo . --json pdm run dev ``` @@ -40,3 +40,28 @@ Report vulnerabilities privately through [SECURITY.md](SECURITY.md). Security de - OpenAPI schema: `docs/openapi.json` (regenerate with `pdm run python scripts/generate-openapi.py`) - This repository does not publish hosted API documentation automatically + +## 独立 Python SDK + +构建 `pdm build` 得到 `inkcre-core` wheel，安装后使用 `inkcre_core`。基础依赖不包含 FastAPI、Uvicorn、APScheduler 或扩展安装器；服务开发使用 `pdm install -G app -G dev`。Core 当前支持 Python 3.12、PostgreSQL/psycopg，数据库 schema 和迁移仍由本仓 App 运维入口管理。 + +```python +from inkcre_core import Core +from inkcre_core.business.info_base import BlockService + +async def read_block(database_url: str, block_id: int): + core = Core(database_url) + try: + with core.bind(): + return await BlockService.get(block_id) + finally: + await core.aclose() +``` + +`Core` 构造不连接数据库或启动执行者。调用方传入数据库 URL 或借用的 `AsyncEngine`，用 `CoreOptions` 提供 Peer 身份、认证和调试参数。绑定按异步任务隔离，子任务继承；每次领域操作仍建立自己的短事务。A、B 实例可以并行操作不同信息库，应用切换配置不改变既有实例。关闭会收束该实例的 Job，释放自建 engine；借用 engine 的关闭权仍属于调用方。调用方应先结束自己的操作再关闭实例。 + +公开消费面是 `Core` / `CoreOptions`、`inkcre_core.schemas` 的模型，以及各 `inkcre_core.business` 包导出的领域入口（例如 `BlockService`、`submit_graph`）。`persistence` 和私有实现模块属于内部细节。普通读写不需要启动 Job、恢复扩展或加载 `app.settings`；使用 Resolver 时可显式注册所需实现。Job 的启动和轮询由调用方决定。 + +HTTP 路由、环境读取、调度频率、扩展安装／进程重启和 Sink 的服务挂载由 `app/` 保留。已有独立 Extension runtime 的生命周期合同保持原 owner。原 `app.business.*`、`app.schemas.*`、`libs.*`、`utils.*` 导入作为服务的兼容入口保留，历史迁移与已发布扩展仍指向同一实现；独立 wheel 只交付 `inkcre_core`，不交付这些旧服务路径。 + +本地边界检查：`pdm run check:core-package` 构建 wheel 并在仓库之外检查独立消费，`pdm run python -m pytest -q tests/test_core_context.py` 验证并行绑定与资源所有权；两者均不连接数据库，不属于端到端验收。 diff --git a/app/AGENTS.md b/app/AGENTS.md index d3ebabc2..797ef93c 100644 --- a/app/AGENTS.md +++ b/app/AGENTS.md @@ -1,9 +1,9 @@ -# Application Core +# Service Application > Applies to `app/`; deeper guides add narrower constraints. - Authority and dependency direction are documented in `docs/30-unit-tdd/business-pipeline-and-authority.md`; do not reconstruct them from directory names. -- Settings enter through `app/settings.py`; database sessions enter through `app/engine.py`. Do not create a second process-global settings, engine, or session authority. +- Settings enter through `app/settings.py`; `app/core.py` composes and binds the service-owned SDK instance. `app/engine.py` only preserves old service imports. Domain operations use the bound Core and own short UoWs. - Runtime registration and database effects belong in lifespan/bootstrap paths, not module import. Preserve import-only OpenAPI generation. - Routes parse transport input and call owning business managers; do not duplicate business policy in routes. - Required check: run the narrow tests for the changed unit, then `pdm run check:foundation` for bootstrap, dependency, or import-boundary changes. diff --git a/app/business/AGENTS.md b/app/business/AGENTS.md index a6d9d538..c6bc92b7 100644 --- a/app/business/AGENTS.md +++ b/app/business/AGENTS.md @@ -1,12 +1,3 @@ -# Business Units +# Service-owned execution -> Applies to `app/business/`; deeper guides own local hazards. - -- Read `docs/30-unit-tdd/business-pipeline-and-authority.md` before a change crosses business subtrees. -- `info_base` owns graph persistence; sources and extensions propose commands but do not copy generic persistence. -- AI execution remains graph-blind. Agent and organization layers may compose it but must not move graph or resolver policy into AI. -- Semantic and lexical retrieval own separate derived records and ranking policy; neither owns graph facts or mutates the graph while reading. -- Global Job/Cron machinery owns durable execution occurrence and claim lifecycle; domain units retain typed payload and effect semantics. -- Peer owns identity, capability, lease, and delegation transport facts; typed business codecs remain with their business owner. -- Application use cases choose UoW scopes; raw sessions stay in persistence. Composed operations require a UoW and never end its transaction. Each concurrent task owns a separate UoW; external I/O runs outside DB scopes. -- Required check: run tests for every affected business owner and the repository import-boundary checks. +Extension installation/restart and FastAPI Sink composition belong here. Reusable domain operations and transaction ownership live in `inkcre_core`; remaining old domain modules only preserve published imports. Do not add application settings, scheduler or HTTP dependencies to the SDK. diff --git a/app/business/agent/__init__.py b/app/business/agent/__init__.py index 33fcb1d3..a891e654 100644 --- a/app/business/agent/__init__.py +++ b/app/business/agent/__init__.py @@ -1,40 +1,3 @@ -"""Reusable Agent definitions, typed Tools and in-memory Thread execution.""" +"""Legacy import; use inkcre_core for new consumers.""" -from .contracts import ( - AgentError, - AgentNotFoundError, - AgentToolBindingError, - AgentTurnActiveError, - BoundAgentTool, - DuplicateAgentToolRegistrationError, - MissingAgentToolError, - ToolExecutionError, -) -from .main import AgentManager -from .bootstrap import register_core_agent_tools -from .persistence import ( - InMemoryThreadPersistenceBackend, - ThreadID, - ThreadPersistenceBackend, - ThreadState, -) -from .thread import Thread, TurnTermination - -__all__ = [ - "AgentError", - "AgentManager", - "register_core_agent_tools", - "AgentNotFoundError", - "AgentToolBindingError", - "AgentTurnActiveError", - "BoundAgentTool", - "DuplicateAgentToolRegistrationError", - "InMemoryThreadPersistenceBackend", - "MissingAgentToolError", - "Thread", - "ThreadID", - "ThreadPersistenceBackend", - "ThreadState", - "ToolExecutionError", - "TurnTermination", -] +from inkcre_core.business.agent import * diff --git a/app/business/agent/bootstrap.py b/app/business/agent/bootstrap.py index d55bff50..c97d869a 100644 --- a/app/business/agent/bootstrap.py +++ b/app/business/agent/bootstrap.py @@ -1,17 +1,10 @@ -"""Explicit registration entrypoint for core-owned Agent Tool controllers.""" +"""Legacy import; use inkcre_core for new consumers.""" +from typing import TYPE_CHECKING +from importlib import import_module +import sys -def register_core_agent_tools() -> None: - """Import every core Tool owner, including Sink delivery controllers. +sys.modules[__name__] = import_module("inkcre_core.business.agent.bootstrap") - Decorators register Tools when their modules load; Python's import cache makes - repeated bootstrap calls harmless. Entry points must call this explicitly, - rather than rely on routes or package re-exports importing the owners first. - """ - from app.business.graph_navigation_retrieval import tools as _graph_tools - from app.business.info_base import tools as _info_base_tools - from app.business.info_base.resolver import tools as _resolver_tools - from app.business.organization import tools as _organization_tools - from app.business.sink import agent_query as _agent_query - - del _graph_tools, _info_base_tools, _resolver_tools, _organization_tools, _agent_query +if TYPE_CHECKING: + from inkcre_core.business.agent.bootstrap import * diff --git a/app/business/agent/contracts.py b/app/business/agent/contracts.py index 6b47fdc9..26349453 100644 --- a/app/business/agent/contracts.py +++ b/app/business/agent/contracts.py @@ -1,59 +1,10 @@ -"""Agent Tool binding and runtime failure contracts.""" +"""Legacy import; use inkcre_core for new consumers.""" -from __future__ import annotations +from typing import TYPE_CHECKING +from importlib import import_module +import sys -from collections.abc import Awaitable, Callable -from dataclasses import dataclass -import typing +sys.modules[__name__] = import_module("inkcre_core.business.agent.contracts") -import pydantic - -from app.schemas.ai import FunctionTool, JSONValue - - -AgentToolHandler: typing.TypeAlias = Callable[ - [pydantic.BaseModel], - JSONValue | Awaitable[JSONValue], -] -AgentToolInputModelFactory: typing.TypeAlias = Callable[[], type[pydantic.BaseModel]] - - -class AgentError(RuntimeError): - """Base failure exposed by the Agent domain.""" - - -class AgentNotFoundError(AgentError): - pass - - -class AgentToolBindingError(AgentError): - pass - - -class DuplicateAgentToolRegistrationError(AgentToolBindingError): - pass - - -class MissingAgentToolError(AgentToolBindingError): - pass - - -class AgentTurnActiveError(AgentError): - pass - - -class ToolExecutionError(Exception): - """A Tool-owned actionable failure intended for the model.""" - - def __init__(self, content: JSONValue): - super().__init__("Agent Tool execution failed") - self.content = content - - -@dataclass(frozen=True) -class BoundAgentTool: - """One run-local Tool schema, validation model and code-owned handler.""" - - definition: FunctionTool - input_model: type[pydantic.BaseModel] - handler: AgentToolHandler +if TYPE_CHECKING: + from inkcre_core.business.agent.contracts import * diff --git a/app/business/agent/debug.py b/app/business/agent/debug.py index deeb4239..a9ecc8ce 100644 --- a/app/business/agent/debug.py +++ b/app/business/agent/debug.py @@ -1,51 +1,10 @@ -"""Opt-in development traces through the existing logging backend.""" +"""Legacy import; use inkcre_core for new consumers.""" -import asyncio -import datetime -import json +from typing import TYPE_CHECKING +from importlib import import_module import sys -import typing -import uuid -import pydantic +sys.modules[__name__] = import_module("inkcre_core.business.agent.debug") -from app.settings import settings -from libs.obsrv.log_record import ENABLE_LOG_BACKEND, TRACE_ID -from libs.obsrv.main import get_logger - - -def _json_value(value: typing.Any) -> typing.Any: - if isinstance(value, pydantic.BaseModel): - return value.model_dump(mode="python") - if isinstance(value, bytes): - return {"omitted_binary_bytes": len(value)} - if isinstance(value, datetime.datetime | datetime.date): - return value.isoformat() - if isinstance(value, uuid.UUID): - return str(value) - raise TypeError(f"Unsupported Agent trace value: {type(value).__name__}") - - -def _emit(event: str, thread_id: str, payload: dict[str, typing.Any]) -> None: - token = ENABLE_LOG_BACKEND.set(True) - try: - envelope = { - "event": event, - "thread_id": thread_id, - "trace_id": TRACE_ID.get(), - **payload, - } - get_logger().getChild("agent.debug").info( - json.dumps(envelope, ensure_ascii=False, default=_json_value), - extra={"event": event, "agent_thread_id": thread_id}, - ) - except Exception as error: - # A debug destination must never replace the Agent's real outcome. - sys.stderr.write(f"Agent debug trace unavailable: {type(error).__name__}\n") - finally: - ENABLE_LOG_BACKEND.reset(token) - - -async def trace(event: str, thread_id: uuid.UUID, **payload: typing.Any) -> None: - if settings.obsrv.agent_debug: - await asyncio.to_thread(_emit, event, str(thread_id), payload) +if TYPE_CHECKING: + from inkcre_core.business.agent.debug import * diff --git a/app/business/agent/main.py b/app/business/agent/main.py index 94f2e20f..cb79293a 100644 --- a/app/business/agent/main.py +++ b/app/business/agent/main.py @@ -1,244 +1,10 @@ -"""Persisted Agent definition loading and exact Tool registry binding.""" +"""Legacy import; use inkcre_core for new consumers.""" -from __future__ import annotations +from typing import TYPE_CHECKING +from importlib import import_module +import sys -from dataclasses import dataclass -import inspect -import typing +sys.modules[__name__] = import_module("inkcre_core.business.agent.main") -import pydantic - -from app.business.ai import AIExecutionRequirement, AIManager -from app.schemas import AgentDefinitionModel -from app.schemas.agent import AgentID, AgentForm, AgentUpdateForm -from app.schemas.ai import FunctionTool, SystemMessage, UserMessage - -from app.persistence.agent.uow import agent_uow - -from .contracts import ( - AgentNotFoundError, - AgentToolBindingError, - AgentToolHandler, - AgentToolInputModelFactory, - BoundAgentTool, - DuplicateAgentToolRegistrationError, - MissingAgentToolError, -) -from .persistence import ( - InMemoryThreadPersistenceBackend, - ThreadPersistenceBackend, - ThreadState, -) -from .thread import Thread -from .debug import trace - - -HandlerT = typing.TypeVar("HandlerT", bound=typing.Callable[..., typing.Any]) - - -@dataclass(frozen=True) -class _ToolRegistration: - handler: AgentToolHandler - original_handler: typing.Callable[..., typing.Any] - description: str - input_model: type[pydantic.BaseModel] - input_model_factory: AgentToolInputModelFactory | None - - def bind(self, tool_id: str) -> BoundAgentTool: - input_model = ( - self.input_model_factory() - if self.input_model_factory is not None - else self.input_model - ) - if not isinstance(input_model, type) or not issubclass(input_model, pydantic.BaseModel): - raise AgentToolBindingError( - f"Agent Tool {tool_id!r} did not produce a Pydantic input model" - ) - return BoundAgentTool( - definition=FunctionTool( - id=tool_id, - description=self.description, - input_schema=input_model.model_json_schema(), - ), - input_model=input_model, - handler=self.handler, - ) - - -class AgentManager: - """Agent-definition execution and peer-local exact Tool registry.""" - - _TOOLS: dict[str, _ToolRegistration] = {} - _persistence: ThreadPersistenceBackend = InMemoryThreadPersistenceBackend() - - @classmethod - async def get_definition(cls, agent_id: AgentID) -> AgentDefinitionModel | None: - async with agent_uow() as agents: - return await agents.get(agent_id) - - @classmethod - async def list_definitions( - cls, *, limit: int | None = None, cursor: int | None = None - ) -> tuple[list[AgentDefinitionModel], int | None]: - async with agent_uow() as agents: - return await agents.list(limit=limit, cursor=cursor) - - @classmethod - async def create_definition(cls, form: AgentForm) -> AgentDefinitionModel: - async with agent_uow() as agents: - record = await agents.create(form) - return record - - @classmethod - async def update_definition( - cls, agent_id: AgentID, form: AgentUpdateForm - ) -> AgentDefinitionModel: - async with agent_uow() as agents: - record = await agents.get(agent_id, for_update=True) - if record is None: - raise AgentNotFoundError(f"Agent {agent_id} does not exist") - changes = form.model_dump(exclude_unset=True) - candidate = AgentForm.model_validate( - {**{field: getattr(record, field) for field in AgentForm.model_fields}, **changes} - ) - for field in changes: - setattr(record, field, getattr(candidate, field)) - await agents.save(record) - return record - - @classmethod - async def delete_definition(cls, agent_id: AgentID) -> bool: - async with agent_uow() as agents: - record = await agents.get(agent_id) - if record is None: - return False - await agents.delete(record) - return True - - @classmethod - def list_tools( - cls, *, limit: int | None = None, cursor: str | None = None - ) -> tuple[list[dict[str, str]], str | None]: - ids = sorted(key for key in cls._TOOLS if cursor is None or key > cursor) - more = limit is not None and len(ids) > limit - ids = ids[:limit] - return [{"id": key, "description": cls._TOOLS[key].description} for key in ids], ids[ - -1 - ] if more else None - - @classmethod - def get_tool(cls, tool_id: str) -> FunctionTool: - registration = cls._TOOLS.get(tool_id) - if registration is None: - raise MissingAgentToolError(f"Agent Tool {tool_id!r} is not registered") - return registration.bind(tool_id).definition - - @classmethod - async def can_execute(cls, agent_id: AgentID, input_modality: str) -> bool: - """Return static local eligibility for one Agent and canonical input modality.""" - definition = await cls.get_definition(agent_id) - if definition is None: - return False - try: - cls._bind_tools(definition.tools) - except MissingAgentToolError: - return False - requires_tools = bool(definition.tools) or definition.tool_choice is not None - return await AIManager.can_execute( - definition.model, - AIExecutionRequirement( - capability="chat", - input_modalities=frozenset({"text", input_modality}), - output_modalities=frozenset({"text"}), - features=frozenset({"tool_calling"}) if requires_tools else frozenset(), - tool_choice=definition.tool_choice, - ), - ) - - @classmethod - def tool( - cls, - tool_id: str, - *, - description: str, - input_model_factory: AgentToolInputModelFactory | None = None, - ) -> typing.Callable[[HandlerT], HandlerT]: - """Register one function as a typed Agent Tool and return it unchanged.""" - if not tool_id: - raise AgentToolBindingError("Agent Tool ID must not be empty") - - def decorator(handler: HandlerT) -> HandlerT: - signature = inspect.signature(handler) - parameters = tuple(signature.parameters.values()) - if len(parameters) != 1: - raise AgentToolBindingError( - f"Agent Tool {tool_id!r} handler must accept exactly one input model" - ) - hints = typing.get_type_hints(handler) - annotation = hints.get(parameters[0].name) - if not isinstance(annotation, type) or not issubclass(annotation, pydantic.BaseModel): - raise AgentToolBindingError( - f"Agent Tool {tool_id!r} input must be annotated with a Pydantic model" - ) - - existing = cls._TOOLS.get(tool_id) - if existing is not None: - if ( - existing.original_handler is handler - and existing.description == description - and existing.input_model_factory is input_model_factory - ): - return handler - raise DuplicateAgentToolRegistrationError( - f"Agent Tool {tool_id!r} is already registered" - ) - - cls._TOOLS[tool_id] = _ToolRegistration( - handler=typing.cast(AgentToolHandler, handler), - original_handler=handler, - description=description, - input_model=annotation, - input_model_factory=input_model_factory, - ) - return handler - - return decorator - - @classmethod - async def run(cls, agent_id: AgentID, initial_message: UserMessage) -> Thread: - """Create one active Thread from a persisted Agent definition snapshot.""" - definition = await cls.get_definition(agent_id) - if definition is None: - raise AgentNotFoundError(f"Agent {agent_id} does not exist") - - bound_tools = cls._bind_tools(definition.tools) - state = ThreadState( - model=definition.model, - tools=tuple(tool.definition for tool in bound_tools), - tool_choice=definition.tool_choice, - max_model_calls_per_turn=definition.max_model_calls_per_turn, - messages=(SystemMessage(content=definition.system_prompt),), - ) - thread_id, persisted = await cls._persistence.create(state) - await trace( - "agent.thread.created", - thread_id, - agent_id=agent_id, - agent_name=definition.name, - state=persisted, - ) - thread = Thread(thread_id, persisted, cls._persistence, bound_tools) - thread.start_turn(initial_message) - return thread - - @classmethod - def _bind_tools(cls, tool_ids: tuple[str, ...]) -> tuple[BoundAgentTool, ...]: - bound: list[BoundAgentTool] = [] - for tool_id in tool_ids: - registration = cls._TOOLS.get(tool_id) - if registration is None: - raise MissingAgentToolError( - f"Agent definition references unavailable Tool {tool_id!r}" - ) - bound.append(registration.bind(tool_id)) - return tuple(bound) +if TYPE_CHECKING: + from inkcre_core.business.agent.main import * diff --git a/app/business/agent/persistence.py b/app/business/agent/persistence.py index 2b8252bc..a1d43b34 100644 --- a/app/business/agent/persistence.py +++ b/app/business/agent/persistence.py @@ -1,147 +1,10 @@ -"""Replaceable whole-Thread persistence with an in-memory MVP backend.""" +"""Legacy import; use inkcre_core for new consumers.""" -from __future__ import annotations +from typing import TYPE_CHECKING +from importlib import import_module +import sys -import asyncio -from abc import ABC, abstractmethod -import typing -import uuid +sys.modules[__name__] = import_module("inkcre_core.business.agent.persistence") -import pydantic - -from app.schemas.ai import ( - AIModelID, - AssistantMessage, - FunctionTool, - Message, - ToolChoice, - validate_message_history, -) - - -ThreadID: typing.TypeAlias = uuid.UUID - - -def validate_thread_message_history(value: typing.Any) -> tuple[Message, ...]: - """Allow strict history plus one recoverable trailing ToolCall message.""" - messages = tuple(value) - if messages and isinstance(messages[-1], AssistantMessage) and messages[-1].tool_calls: - validate_message_history(messages[:-1]) - return messages - return validate_message_history(messages) - - -class ThreadState(pydantic.BaseModel): - """Serializable Thread snapshot owned by a persistence backend.""" - - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - model: AIModelID - tools: tuple[FunctionTool, ...] - tool_choice: ToolChoice | None - max_model_calls_per_turn: int = pydantic.Field(gt=0) - messages: tuple[Message, ...] - - @pydantic.field_validator("tools") - @classmethod - def unique_tools(cls, value: tuple[FunctionTool, ...]) -> tuple[FunctionTool, ...]: - tool_ids = tuple(tool.id for tool in value) - if len(tool_ids) != len(set(tool_ids)): - raise ValueError("Thread Tool IDs must be unique") - return value - - @pydantic.field_validator("messages") - @classmethod - def closed_or_recoverable_history(cls, value: tuple[Message, ...]) -> tuple[Message, ...]: - return validate_thread_message_history(value) - - -class ThreadPersistenceBackend(ABC): - """Atomic persistence boundary for complete Thread snapshots.""" - - @abstractmethod - async def create(self, state: ThreadState) -> tuple[ThreadID, ThreadState]: - raise NotImplementedError - - @abstractmethod - async def read(self, thread_id: ThreadID) -> ThreadState: - raise NotImplementedError - - @abstractmethod - async def append( - self, - thread_id: ThreadID, - messages: tuple[Message, ...], - ) -> ThreadState: - raise NotImplementedError - - @abstractmethod - async def discard_trailing_incomplete_tool_calls( - self, - thread_id: ThreadID, - ) -> ThreadState: - raise NotImplementedError - - -class InMemoryThreadPersistenceBackend(ThreadPersistenceBackend): - """Process-local backend preserving atomic Thread message updates.""" - - def __init__(self) -> None: - self._states: dict[ThreadID, ThreadState] = {} - self._locks: dict[ThreadID, asyncio.Lock] = {} - - async def create(self, state: ThreadState) -> tuple[ThreadID, ThreadState]: - thread_id = uuid.uuid4() - self._states[thread_id] = state - self._locks[thread_id] = asyncio.Lock() - return thread_id, state - - async def read(self, thread_id: ThreadID) -> ThreadState: - try: - return self._states[thread_id] - except KeyError as error: - raise KeyError(f"Thread {thread_id} does not exist") from error - - async def append( - self, - thread_id: ThreadID, - messages: tuple[Message, ...], - ) -> ThreadState: - async with self._lock(thread_id): - current = await self.read(thread_id) - updated = ThreadState( - model=current.model, - tools=current.tools, - tool_choice=current.tool_choice, - max_model_calls_per_turn=current.max_model_calls_per_turn, - messages=current.messages + messages, - ) - self._states[thread_id] = updated - return updated - - async def discard_trailing_incomplete_tool_calls( - self, - thread_id: ThreadID, - ) -> ThreadState: - async with self._lock(thread_id): - current = await self.read(thread_id) - if not current.messages: - return current - trailing = current.messages[-1] - if not isinstance(trailing, AssistantMessage) or not trailing.tool_calls: - return current - updated = ThreadState( - model=current.model, - tools=current.tools, - tool_choice=current.tool_choice, - max_model_calls_per_turn=current.max_model_calls_per_turn, - messages=current.messages[:-1], - ) - self._states[thread_id] = updated - return updated - - def _lock(self, thread_id: ThreadID) -> asyncio.Lock: - try: - return self._locks[thread_id] - except KeyError as error: - raise KeyError(f"Thread {thread_id} does not exist") from error +if TYPE_CHECKING: + from inkcre_core.business.agent.persistence import * diff --git a/app/business/agent/projection.py b/app/business/agent/projection.py index 658b2a5d..7c57cb83 100644 --- a/app/business/agent/projection.py +++ b/app/business/agent/projection.py @@ -1,45 +1,10 @@ -"""JSON projection shared by domain-owned Agent Tool controllers.""" +"""Legacy import; use inkcre_core for new consumers.""" -import dataclasses -import typing +from typing import TYPE_CHECKING +from importlib import import_module +import sys -import pydantic +sys.modules[__name__] = import_module("inkcre_core.business.agent.projection") -from app.schemas.ai import JSONValue - - -_JSON_ADAPTER = pydantic.TypeAdapter(JSONValue) - - -def project_json(value: typing.Any) -> JSONValue: - """Project ordinary domain values without admitting binary Tool results.""" - if _contains_bytes(value): - raise TypeError("Binary values are unavailable through JSON Agent Tools") - if isinstance(value, pydantic.BaseModel): - projected = value.model_dump(mode="json") - elif dataclasses.is_dataclass(value) and not isinstance(value, type): - projected = dataclasses.asdict(value) - else: - projected = pydantic.TypeAdapter(typing.Any).dump_python( - typing.cast(typing.Any, value), - mode="json", - ) - return _JSON_ADAPTER.validate_python(projected) - - -def _contains_bytes(value: typing.Any) -> bool: - if isinstance(value, bytes): - return True - if isinstance(value, pydantic.BaseModel): - return any( - _contains_bytes(getattr(value, field)) for field in value.__class__.model_fields - ) - if dataclasses.is_dataclass(value) and not isinstance(value, type): - return any( - _contains_bytes(getattr(value, field.name)) for field in dataclasses.fields(value) - ) - if isinstance(value, dict): - return any(_contains_bytes(item) for item in value.values()) - if isinstance(value, list | tuple | set | frozenset): - return any(_contains_bytes(item) for item in value) - return False +if TYPE_CHECKING: + from inkcre_core.business.agent.projection import * diff --git a/app/business/agent/thread.py b/app/business/agent/thread.py index d8d06c91..83f9ca75 100644 --- a/app/business/agent/thread.py +++ b/app/business/agent/thread.py @@ -1,316 +1,10 @@ -"""Cancellable Agent Thread and per-turn model/Tool orchestration.""" +"""Legacy import; use inkcre_core for new consumers.""" -from __future__ import annotations +from typing import TYPE_CHECKING +from importlib import import_module +import sys -import asyncio -from enum import StrEnum -import inspect -import hashlib -import json -import time -import traceback -import typing +sys.modules[__name__] = import_module("inkcre_core.business.agent.thread") -import pydantic - -from app.business.ai import AIManager -from app.schemas.ai import ( - AssistantMessage, - JSONValue, - ToolCall, - ToolResult, - ToolResultMessage, - UserMessage, -) - -from .contracts import AgentTurnActiveError, BoundAgentTool, ToolExecutionError -from .persistence import ThreadID, ThreadPersistenceBackend, ThreadState -from .debug import trace -from libs.obsrv.main import get_logger -from libs.obsrv.telemetry import operation - - -logger = get_logger().getChild("agent.thread") - - -class TurnTermination(StrEnum): - COMPLETED = "completed" - MAX_MODEL_CALLS = "max_model_calls" - - -class Thread: - """One persisted Agent-definition snapshot with at most one active Turn.""" - - def __init__( - self, - thread_id: ThreadID, - state: ThreadState, - persistence: ThreadPersistenceBackend, - tools: tuple[BoundAgentTool, ...], - ) -> None: - self.id = thread_id - self._state = state - self._persistence = persistence - self._tools = {tool.definition.id: tool for tool in tools} - self.current_turn: asyncio.Task[TurnTermination] | None = None - self._turn_index = 0 - self._model_calls = 0 - - @property - def messages(self): - return self._state.messages - - @property - def model(self): - return self._state.model - - @property - def tools(self): - return self._state.tools - - @property - def tool_choice(self): - return self._state.tool_choice - - @property - def max_model_calls_per_turn(self): - return self._state.max_model_calls_per_turn - - async def refresh(self) -> ThreadState: - """Refresh the local view from the persistence authority.""" - self._state = await self._persistence.read(self.id) - return self._state - - def start_turn(self, input: UserMessage) -> asyncio.Task[TurnTermination]: - """Schedule one complete model/Tool turn and return its cancellation handle.""" - if self.current_turn is not None and not self.current_turn.done(): - raise AgentTurnActiveError(f"Thread {self.id} already has an active Turn") - self.current_turn = asyncio.create_task( - self._run_turn(input), - name=f"agent-thread-{self.id}", - ) - return self.current_turn - - async def _run_turn(self, input: UserMessage) -> TurnTermination: - self._turn_index += 1 - self._model_calls = 0 - with operation( - "agent.turn", - attributes={ - "gen_ai.conversation.id": str(self.id), - "inkcre.agent.turn": self._turn_index, - }, - ) as observation: - span = observation.span - started = time.monotonic() - await trace( - "agent.turn.started", - self.id, - turn=self._turn_index, - input=input, - model=self.model, - max_model_calls=self.max_model_calls_per_turn, - ) - try: - outcome = await self._execute_turn(input) - except asyncio.CancelledError: - span.set_attribute("inkcre.agent.outcome", "cancelled") - await trace( - "agent.turn.finished", - self.id, - turn=self._turn_index, - model_calls=self._model_calls, - outcome="cancelled", - elapsed_seconds=time.monotonic() - started, - ) - raise - except Exception as error: - span.set_attribute("inkcre.agent.outcome", "failed") - await trace( - "agent.turn.finished", - self.id, - turn=self._turn_index, - model_calls=self._model_calls, - outcome="failed", - error_type=type(error).__name__, - error=str(error), - traceback=traceback.format_exc(), - elapsed_seconds=time.monotonic() - started, - ) - raise - await trace( - "agent.turn.finished", - self.id, - turn=self._turn_index, - model_calls=self._model_calls, - outcome=outcome, - elapsed_seconds=time.monotonic() - started, - ) - span.set_attribute("inkcre.agent.outcome", outcome.value) - span.set_attribute("inkcre.agent.model_calls", self._model_calls) - return outcome - - async def _execute_turn(self, input: UserMessage) -> TurnTermination: - self._state = await self._persistence.discard_trailing_incomplete_tool_calls(self.id) - self._state = await self._persistence.append(self.id, (input,)) - - while True: - self._model_calls += 1 - with operation( - "agent.step", - attributes={ - "gen_ai.conversation.id": str(self.id), - "inkcre.agent.turn": self._turn_index, - "inkcre.agent.model_call": self._model_calls, - }, - ): - await trace( - "agent.model.started", - self.id, - turn=self._turn_index, - call=self._model_calls, - ) - started = time.monotonic() - assistant = await AIManager.chat( - self._state.model, - self._state.messages, - self._state.tools, - self._state.tool_choice, - ) - await trace( - "agent.model.completed", - self.id, - turn=self._turn_index, - call=self._model_calls, - response=assistant, - elapsed_seconds=time.monotonic() - started, - ) - if not assistant.tool_calls: - self._state = await self._persistence.append(self.id, (assistant,)) - return TurnTermination.COMPLETED - - results = await self._execute_tool_batch(assistant) - self._state = await self._persistence.append( - self.id, - (assistant, ToolResultMessage(results=results)), - ) - if self._model_calls >= self._state.max_model_calls_per_turn: - return TurnTermination.MAX_MODEL_CALLS - - async def _execute_tool_batch( - self, - assistant: AssistantMessage, - ) -> tuple[ToolResult, ...]: - tasks = tuple( - asyncio.create_task( - self._execute_tool_call(call), - name=f"agent-tool-{call.tool}-{call.id}", - ) - for call in assistant.tool_calls - ) - return tuple(await asyncio.gather(*tasks)) - - async def _execute_tool_call(self, call: ToolCall) -> ToolResult: - with operation( - "agent.tool", - attributes={ - "gen_ai.operation.name": "execute_tool", - "gen_ai.conversation.id": str(self.id), - "inkcre.agent.turn": self._turn_index, - "inkcre.agent.model_call": self._model_calls, - # Provider-generated IDs may contain content; retain correlation as a digest. - "inkcre.agent.tool_call.id_hash": hashlib.sha256( - call.id.encode(errors="replace") - ).hexdigest(), - }, - ) as observation: - span = observation.span - tool = self._tools.get(call.tool) - if tool is not None: - span.set_attribute("gen_ai.tool.name", tool.definition.id) - await trace( - "agent.tool.started", - self.id, - turn=self._turn_index, - call=self._model_calls, - tool_call=call, - ) - started = time.monotonic() - try: - result = await self._invoke_tool_call(call) - except asyncio.CancelledError: - span.set_attribute("inkcre.agent.outcome", "cancelled") - await trace( - "agent.tool.cancelled", - self.id, - turn=self._turn_index, - call=self._model_calls, - tool_call_id=call.id, - tool=call.tool, - elapsed_seconds=time.monotonic() - started, - ) - raise - await trace( - "agent.tool.completed", - self.id, - turn=self._turn_index, - call=self._model_calls, - tool=call.tool, - result=result, - elapsed_seconds=time.monotonic() - started, - ) - span.set_attribute( - "inkcre.agent.outcome", "error" if result.is_error else "completed" - ) - if result.is_error: - span.set_attribute("error.type", "ToolResultError") - observation.outcome = "error" - return result - - async def _invoke_tool_call(self, call: ToolCall) -> ToolResult: - tool = self._tools.get(call.tool) - if tool is None: - return ToolResult( - tool_call_id=call.id, - content={"error": "unknown_tool", "tool": call.tool}, - is_error=True, - ) - - try: - tool_input = tool.input_model.model_validate(call.arguments) - except pydantic.ValidationError as error: - return ToolResult( - tool_call_id=call.id, - content=typing.cast(JSONValue, json.loads(error.json())), - is_error=True, - ) - - try: - content = tool.handler(tool_input) - if inspect.isawaitable(content): - content = await content - return ToolResult(tool_call_id=call.id, content=content) - except ToolExecutionError as error: - return ToolResult( - tool_call_id=call.id, - content=error.content, - is_error=True, - ) - except Exception as error: - await trace( - "agent.tool.exception", - self.id, - turn=self._turn_index, - call=self._model_calls, - tool_call_id=call.id, - tool=call.tool, - error_type=type(error).__name__, - error=str(error), - traceback=traceback.format_exc(), - ) - logger.exception("Unexpected Agent Tool failure", extra={"tool": call.tool}) - return ToolResult( - tool_call_id=call.id, - content={"error": "tool_execution_failed"}, - is_error=True, - ) +if TYPE_CHECKING: + from inkcre_core.business.agent.thread import * diff --git a/app/business/ai/__init__.py b/app/business/ai/__init__.py index df830957..a0700833 100644 --- a/app/business/ai/__init__.py +++ b/app/business/ai/__init__.py @@ -1,45 +1,3 @@ -"""AI registry facts and peer-local capability execution.""" +"""Legacy import; use inkcre_core for new consumers.""" -from .contracts import ( - AICapabilityUnavailableError, - AIDialectAdapter, - AIExecutionRequirement, - AIFeatureUnavailableError, - AIInputUnavailableError, - AIModelDisabledError, - AIModelNotFoundError, - AIOutputContractError, - AIProviderDisabledError, - AIProviderNotFoundError, - DuplicateAIDialectRegistrationError, - InvalidAIProviderConfigError, - UnknownAIDialectError, -) -from .main import AIManager - -# Import installed core dialects only after AIManager exists so decorators register. -from .dialects import ( - AlibabaModelStudioDialect, - OpenAICompatibleConfig, - OpenAICompatibleDialect, -) - -__all__ = [ - "AICapabilityUnavailableError", - "AIDialectAdapter", - "AIExecutionRequirement", - "AIFeatureUnavailableError", - "AIInputUnavailableError", - "AIManager", - "AIModelDisabledError", - "AIModelNotFoundError", - "AIOutputContractError", - "AIProviderDisabledError", - "AIProviderNotFoundError", - "AlibabaModelStudioDialect", - "DuplicateAIDialectRegistrationError", - "InvalidAIProviderConfigError", - "OpenAICompatibleConfig", - "OpenAICompatibleDialect", - "UnknownAIDialectError", -] +from inkcre_core.business.ai import * diff --git a/app/business/ai/contracts.py b/app/business/ai/contracts.py index f9f3b15a..9a5a3f35 100644 --- a/app/business/ai/contracts.py +++ b/app/business/ai/contracts.py @@ -1,111 +1,10 @@ -"""AI dialect adapter boundary and stable execution failures.""" +"""Legacy import; use inkcre_core for new consumers.""" -import abc -from collections.abc import Sequence -from dataclasses import dataclass +from typing import TYPE_CHECKING +from importlib import import_module +import sys -import pydantic +sys.modules[__name__] = import_module("inkcre_core.business.ai.contracts") -from app.schemas.ai import ( - AICapabilityType, - AssistantMessage, - FunctionTool, - Message, - ToolChoice, -) -from app.schemas.info_base.main import Vector - - -@dataclass(frozen=True) -class AIExecutionRequirement: - """Static capability shape used for peer-local execution eligibility.""" - - capability: AICapabilityType - input_modalities: frozenset[str] - output_modalities: frozenset[str] - features: frozenset[str] = frozenset() - tool_choice: ToolChoice | None = None - - -class DuplicateAIDialectRegistrationError(ValueError): - """Two adapter implementations claimed the same exact dialect ID.""" - - -class UnknownAIDialectError(LookupError): - """The current peer has no adapter for an exact persisted dialect ID.""" - - -class AIProviderNotFoundError(LookupError): - """A requested AI model references no provider visible to this peer.""" - - -class AIModelNotFoundError(LookupError): - """A requested AI model does not exist.""" - - -class AIProviderDisabledError(RuntimeError): - """A disabled provider rejected a new operation.""" - - -class AIModelDisabledError(RuntimeError): - """A disabled model rejected a new operation.""" - - -class InvalidAIProviderConfigError(RuntimeError): - """Persisted provider config does not satisfy its dialect contract.""" - - -class AICapabilityUnavailableError(RuntimeError): - """The selected provider-bound model does not declare an operation.""" - - -class AIFeatureUnavailableError(RuntimeError): - """Effective model/adapter support lacks a requested feature.""" - - -class AIInputUnavailableError(RuntimeError): - """A canonical input cannot be conveyed through the selected dialect.""" - - -class AIOutputContractError(RuntimeError): - """A provider response violated the canonical capability result contract.""" - - -class AIDialectAdapter(abc.ABC): - """Graph-blind provider config/client construction and wire translation.""" - - supported_features: dict[AICapabilityType, frozenset[str]] = {} - supported_input_modalities: dict[AICapabilityType, frozenset[str]] = {} - - @abc.abstractmethod - async def embed( - self, - config: pydantic.BaseModel, - native_model_id: str, - inputs: Sequence[str], - dimensions: int, - ) -> tuple[Vector, ...]: ... - - @abc.abstractmethod - async def chat( - self, - config: pydantic.BaseModel, - native_model_id: str, - messages: Sequence[Message], - tools: Sequence[FunctionTool], - tool_choice: ToolChoice | None, - ) -> AssistantMessage: ... - - def supports_feature(self, capability: AICapabilityType, feature: str) -> bool: - return feature in self.supported_features.get(capability, frozenset()) - - def supports_input_modality( - self, - capability: AICapabilityType, - modality: str, - ) -> bool: - return modality in self.supported_input_modalities.get(capability, frozenset()) - - def supports_tool_choice(self, tool_choice: ToolChoice) -> bool: - del tool_choice - return False +if TYPE_CHECKING: + from inkcre_core.business.ai.contracts import * diff --git a/app/business/ai/dialects/__init__.py b/app/business/ai/dialects/__init__.py index 93f92271..78255c7a 100644 --- a/app/business/ai/dialects/__init__.py +++ b/app/business/ai/dialects/__init__.py @@ -1,10 +1,3 @@ -"""Installed core AI dialect adapters.""" +"""Legacy import; use inkcre_core for new consumers.""" -from .alibaba_model_studio import AlibabaModelStudioDialect -from .openai_compatible import OpenAICompatibleConfig, OpenAICompatibleDialect - -__all__ = [ - "AlibabaModelStudioDialect", - "OpenAICompatibleConfig", - "OpenAICompatibleDialect", -] +from inkcre_core.business.ai.dialects import * diff --git a/app/business/ai/dialects/alibaba_model_studio.py b/app/business/ai/dialects/alibaba_model_studio.py index ee31b56b..f18191f8 100644 --- a/app/business/ai/dialects/alibaba_model_studio.py +++ b/app/business/ai/dialects/alibaba_model_studio.py @@ -1,181 +1,12 @@ -"""Alibaba Model Studio chat dialect, including native multimodal parts.""" +"""Legacy import; use inkcre_core for new consumers.""" -from collections.abc import Sequence -import json -import typing -import time +from typing import TYPE_CHECKING +from importlib import import_module +import sys -from openai import AsyncStream -from openai.types.chat import ChatCompletionChunk -from opentelemetry import trace -from opentelemetry.trace import INVALID_SPAN -import pydantic - -from app.schemas.ai import ( - AudioContentPart, - AssistantMessage, - FunctionTool, - ImageContentPart, - Message, - TextContentPart, - ToolCall, - ToolChoice, - UserMessage, - VideoContentPart, -) - -from libs.obsrv.telemetry import is_enabled - -from ..contracts import AIOutputContractError -from ..main import AIManager -from ..telemetry import record_response_usage -from .openai_compatible import OpenAICompatibleConfig, OpenAICompatibleDialect - - -@AIManager.register_dialect( - "core.alibaba-model-studio.v1", - description="Alibaba Model Studio OpenAI-compatible multimodal chat protocol.", - config_model=OpenAICompatibleConfig, +sys.modules[__name__] = import_module( + "inkcre_core.business.ai.dialects.alibaba_model_studio" ) -class AlibabaModelStudioDialect(OpenAICompatibleDialect): - """Translate canonical chat through Model Studio's multimodal extensions.""" - - supported_features = {"chat": frozenset({"tool_calling"})} - supported_input_modalities = { - "embedding": frozenset({"text"}), - "chat": frozenset({"text", "image", "audio", "video"}), - } - - @classmethod - def _user_content_params(cls, message: UserMessage) -> list[dict[str, typing.Any]]: - result: list[dict[str, typing.Any]] = [] - for part in message.content: - if isinstance(part, TextContentPart): - result.append({"type": "text", "text": part.text}) - elif isinstance(part, ImageContentPart): - result.append( - { - "type": "image_url", - "image_url": { - "url": cls._url_or_inline_data( - mime_type=part.mime_type, - data=part.data, - transfer_url=part.transfer_url, - ) - }, - } - ) - elif isinstance(part, AudioContentPart): - result.append( - { - "type": "input_audio", - "input_audio": { - "data": cls._url_or_inline_data( - mime_type=part.mime_type, - data=part.data, - transfer_url=part.transfer_url, - ), - "format": part.mime_type.removeprefix("audio/"), - }, - } - ) - elif isinstance(part, VideoContentPart): - result.append( - { - "type": "video_url", - "video_url": { - "url": cls._url_or_inline_data( - mime_type=part.mime_type, - data=part.data, - transfer_url=part.transfer_url, - ) - }, - } - ) - return result - - async def chat( - self, - config: pydantic.BaseModel, - native_model_id: str, - messages: Sequence[Message], - tools: Sequence[FunctionTool], - tool_choice: ToolChoice | None, - ) -> AssistantMessage: - client = self._client_factory(self._config(config)) - arguments: dict[str, typing.Any] = { - "model": native_model_id, - "messages": self._message_params(messages), - "stream": True, - "stream_options": {"include_usage": True}, - "modalities": ["text"], - } - if tools: - arguments["tools"] = self._tool_params(tools) - if tool_choice is not None: - arguments["tool_choice"] = self._tool_choice_param(tool_choice) - - span = trace.get_current_span() if is_enabled() else INVALID_SPAN - span.set_attribute("gen_ai.request.stream", True) - started = time.monotonic() - usage = None - finish_reasons: list[str] = [] - first_chunk = True - try: - stream = typing.cast( - AsyncStream[ChatCompletionChunk], - await client.chat.completions.create(**arguments), - ) - text_parts: list[str] = [] - calls: dict[int, dict[str, str]] = {} - async for chunk in stream: - if first_chunk: - span.set_attribute( - "gen_ai.response.time_to_first_chunk", time.monotonic() - started - ) - first_chunk = False - # Usage-only terminal chunks have no choices. Totals replace earlier totals. - if chunk.usage is not None: - usage = chunk.usage - if not chunk.choices: - continue - if chunk.choices[0].finish_reason is not None: - finish_reasons = [chunk.choices[0].finish_reason] - delta = chunk.choices[0].delta - if isinstance(delta.content, str): - text_parts.append(delta.content) - for call in delta.tool_calls or (): - current = calls.setdefault(call.index, {"id": "", "name": "", "arguments": ""}) - if call.id: - current["id"] = call.id - if call.function is not None: - if call.function.name: - current["name"] += call.function.name - if call.function.arguments: - current["arguments"] += call.function.arguments - finally: - record_response_usage("chat", usage, finish_reasons) - await client.close() - parsed_calls: list[ToolCall] = [] - for index in sorted(calls): - call = calls[index] - if not call["id"] or not call["name"]: - raise AIOutputContractError("Alibaba chat stream returned an incomplete ToolCall") - try: - arguments_object = json.loads(call["arguments"]) - except json.JSONDecodeError as error: - raise AIOutputContractError( - f"ToolCall {call['id']!r} arguments are not valid JSON" - ) from error - if not isinstance(arguments_object, dict): - raise AIOutputContractError( - f"ToolCall {call['id']!r} arguments must be a JSON object" - ) - parsed_calls.append( - ToolCall(id=call["id"], tool=call["name"], arguments=arguments_object) - ) - content = "".join(text_parts) or None - if content is None and not parsed_calls: - raise AIOutputContractError("Alibaba chat stream returned no content or ToolCalls") - return AssistantMessage(content=content, tool_calls=tuple(parsed_calls)) +if TYPE_CHECKING: + from inkcre_core.business.ai.dialects.alibaba_model_studio import * diff --git a/app/business/ai/dialects/openai_compatible.py b/app/business/ai/dialects/openai_compatible.py index 8bb24ba8..0f62d1cb 100644 --- a/app/business/ai/dialects/openai_compatible.py +++ b/app/business/ai/dialects/openai_compatible.py @@ -1,323 +1,10 @@ -"""OpenAI-compatible AI dialect across embedding and chat capabilities.""" +"""Legacy import; use inkcre_core for new consumers.""" -import base64 -from collections.abc import Callable, Sequence -import json -import typing +from typing import TYPE_CHECKING +from importlib import import_module +import sys -from openai import AsyncOpenAI -from openai.types.chat import ChatCompletion, ChatCompletionMessageParam -import pydantic +sys.modules[__name__] = import_module("inkcre_core.business.ai.dialects.openai_compatible") -from app.schemas.ai import ( - AudioContentPart, - AssistantMessage, - FunctionTool, - ImageContentPart, - Message, - NamedToolChoice, - SystemMessage, - TextContentPart, - ToolCall, - ToolChoice, - ToolResultMessage, - UserMessage, - VideoContentPart, -) -from app.schemas.info_base.main import Vector - -from ..contracts import AIDialectAdapter, AIInputUnavailableError, AIOutputContractError -from ..main import AIManager -from ..telemetry import record_response_usage - -INLINE_MEDIA_MAX_BYTES = 7 * 1024 * 1024 - - -class OpenAICompatibleConfig(pydantic.BaseModel): - """Connection values for one OpenAI-compatible provider instance.""" - - model_config = pydantic.ConfigDict(extra="forbid") - - api_key: pydantic.SecretStr - base_url: str | None = pydantic.Field(default=None, min_length=1) - - -OpenAIClientFactory: typing.TypeAlias = Callable[[OpenAICompatibleConfig], AsyncOpenAI] - - -@AIManager.register_dialect( - "core.openai-compatible.v1", - description="OpenAI-compatible embedding and chat protocol.", - config_model=OpenAICompatibleConfig, -) -class OpenAICompatibleDialect(AIDialectAdapter): - """Translate canonical AI contracts through the OpenAI Python SDK.""" - - supported_features = {"chat": frozenset({"tool_calling"})} - supported_input_modalities = { - "embedding": frozenset({"text"}), - "chat": frozenset({"text", "image", "audio"}), - } - - def __init__(self, client_factory: OpenAIClientFactory | None = None) -> None: - self._client_factory = client_factory or self._create_client - - @staticmethod - def _create_client(config: OpenAICompatibleConfig) -> AsyncOpenAI: - return AsyncOpenAI( - api_key=config.api_key.get_secret_value(), - base_url=config.base_url, - ) - - @staticmethod - def _config(config: pydantic.BaseModel) -> OpenAICompatibleConfig: - return typing.cast(OpenAICompatibleConfig, config) - - async def embed( - self, - config: pydantic.BaseModel, - native_model_id: str, - inputs: Sequence[str], - dimensions: int, - ) -> tuple[Vector, ...]: - client = self._client_factory(self._config(config)) - usage = None - try: - response = await client.embeddings.create( - model=native_model_id, - input=tuple(inputs), - dimensions=dimensions, - encoding_format="float", - ) - usage = getattr(response, "usage", None) - finally: - record_response_usage("embeddings", usage) - await client.close() - - by_index = {item.index: tuple(item.embedding) for item in response.data} - expected = set(range(len(inputs))) - if set(by_index) != expected: - raise AIOutputContractError( - "OpenAI-compatible embedding response indices do not cover the input batch" - ) - return tuple(by_index[index] for index in range(len(inputs))) - - @staticmethod - def _data_url(mime_type: str, data: bytes) -> str: - encoded = base64.b64encode(data).decode("ascii") - return f"data:{mime_type};base64,{encoded}" - - @classmethod - def _url_or_inline_data( - cls, - *, - mime_type: str, - data: bytes, - transfer_url: str | None, - ) -> str: - if len(data) <= INLINE_MEDIA_MAX_BYTES: - return cls._data_url(mime_type, data) - if transfer_url is not None: - return transfer_url - raise AIInputUnavailableError( - "Media exceeds the inline transfer bound and has no transfer URL" - ) - - @staticmethod - def _audio_format(mime_type: str) -> str: - formats = { - "audio/mpeg": "mp3", - "audio/mp3": "mp3", - "audio/wav": "wav", - "audio/x-wav": "wav", - } - try: - return formats[mime_type] - except KeyError as error: - raise AIInputUnavailableError( - f"OpenAI Chat input_audio does not support {mime_type!r}" - ) from error - - @classmethod - def _user_content_params(cls, message: UserMessage) -> list[dict[str, typing.Any]]: - result: list[dict[str, typing.Any]] = [] - for part in message.content: - if isinstance(part, TextContentPart): - result.append({"type": "text", "text": part.text}) - elif isinstance(part, ImageContentPart): - result.append( - { - "type": "image_url", - "image_url": { - "url": cls._url_or_inline_data( - mime_type=part.mime_type, - data=part.data, - transfer_url=part.transfer_url, - ) - }, - } - ) - elif isinstance(part, AudioContentPart): - if len(part.data) > INLINE_MEDIA_MAX_BYTES: - raise AIInputUnavailableError( - "OpenAI Chat input_audio cannot use a transfer URL and exceeds the " - "inline transfer bound" - ) - result.append( - { - "type": "input_audio", - "input_audio": { - "data": base64.b64encode(part.data).decode("ascii"), - "format": cls._audio_format(part.mime_type), - }, - } - ) - elif isinstance(part, VideoContentPart): - raise AIInputUnavailableError( - "OpenAI Chat does not define a canonical video input part" - ) - return result - - @classmethod - def _message_params(cls, messages: Sequence[Message]) -> list[ChatCompletionMessageParam]: - result: list[ChatCompletionMessageParam] = [] - for message in messages: - if isinstance(message, SystemMessage): - result.append({"role": "system", "content": message.content}) - elif isinstance(message, UserMessage): - result.append( - typing.cast( - ChatCompletionMessageParam, - {"role": "user", "content": cls._user_content_params(message)}, - ) - ) - elif isinstance(message, AssistantMessage): - result.append( - { - "role": "assistant", - "content": message.content, - "tool_calls": [ - { - "id": call.id, - "type": "function", - "function": { - "name": call.tool, - "arguments": json.dumps( - call.arguments, - separators=(",", ":"), - sort_keys=True, - ), - }, - } - for call in message.tool_calls - ], - } - ) - elif isinstance(message, ToolResultMessage): - result.extend( - { - "role": "tool", - "tool_call_id": tool_result.tool_call_id, - "content": json.dumps( - { - "content": tool_result.content, - "is_error": tool_result.is_error, - }, - separators=(",", ":"), - sort_keys=True, - ), - } - for tool_result in message.results - ) - return result - - @staticmethod - def _tool_params(tools: Sequence[FunctionTool]) -> list[dict[str, typing.Any]]: - return [ - { - "type": "function", - "function": { - "name": tool.id, - "description": tool.description, - "parameters": tool.input_schema, - }, - } - for tool in tools - ] - - @staticmethod - def _tool_choice_param(tool_choice: ToolChoice) -> typing.Any: - if isinstance(tool_choice, NamedToolChoice): - return { - "type": "function", - "function": {"name": tool_choice.tool}, - } - return tool_choice - - def supports_tool_choice(self, tool_choice: ToolChoice) -> bool: - return isinstance(tool_choice, NamedToolChoice) or tool_choice in { - "none", - "auto", - "required", - } - - @staticmethod - def _assistant_message(message: typing.Any) -> AssistantMessage: - calls: list[ToolCall] = [] - for call in message.tool_calls or (): - if call.type != "function": - raise AIOutputContractError( - f"Unsupported OpenAI-compatible ToolCall type: {call.type!r}" - ) - try: - parsed_arguments = json.loads(call.function.arguments) - except (TypeError, json.JSONDecodeError) as error: - raise AIOutputContractError( - f"ToolCall {call.id!r} arguments are not valid JSON" - ) from error - if not isinstance(parsed_arguments, dict): - raise AIOutputContractError(f"ToolCall {call.id!r} arguments must be a JSON object") - calls.append( - ToolCall( - id=call.id, - tool=call.function.name, - arguments=parsed_arguments, - ) - ) - return AssistantMessage(content=message.content, tool_calls=tuple(calls)) - - async def chat( - self, - config: pydantic.BaseModel, - native_model_id: str, - messages: Sequence[Message], - tools: Sequence[FunctionTool], - tool_choice: ToolChoice | None, - ) -> AssistantMessage: - client = self._client_factory(self._config(config)) - arguments: dict[str, typing.Any] = { - "model": native_model_id, - "messages": self._message_params(messages), - "stream": False, - } - if tools: - arguments["tools"] = self._tool_params(tools) - if tool_choice is not None: - arguments["tool_choice"] = self._tool_choice_param(tool_choice) - - usage = None - finish_reasons: list[str] = [] - try: - response = typing.cast( - ChatCompletion, - await client.chat.completions.create(**arguments), - ) - usage = response.usage - finish_reasons = [choice.finish_reason for choice in response.choices] - finally: - record_response_usage("chat", usage, finish_reasons) - await client.close() - - if not response.choices: - raise AIOutputContractError("OpenAI-compatible chat response has no choices") - return self._assistant_message(response.choices[0].message) +if TYPE_CHECKING: + from inkcre_core.business.ai.dialects.openai_compatible import * diff --git a/app/business/ai/main.py b/app/business/ai/main.py index afe888ca..97a5d190 100644 --- a/app/business/ai/main.py +++ b/app/business/ai/main.py @@ -1,396 +1,10 @@ -"""Shared AI fact management and peer-local capability routing.""" +"""Legacy import; use inkcre_core for new consumers.""" -from collections.abc import Sequence -from dataclasses import dataclass -import math -import typing +from typing import TYPE_CHECKING +from importlib import import_module +import sys -import pydantic -from opentelemetry.trace import SpanKind +sys.modules[__name__] = import_module("inkcre_core.business.ai.main") -from app.database_contract.profile import BUILTIN_AI_DIALECTS_BY_ID -from app.schemas.ai import ( - AICapabilityType, - AIDialectID, - AIModelCapability, - AIModelID, - AIModelModel, - AIProviderModel, - AssistantMessage, - FunctionTool, - ImageContentPart, - Message, - AudioContentPart, - TextContentPart, - ToolChoice, - UserMessage, - VideoContentPart, - validate_message_history, -) -from app.schemas.info_base.main import Vector - -from app.persistence.ai.uow import ai_uow -from libs.obsrv.telemetry import operation - -from .telemetry import model_attributes - -from .contracts import ( - AICapabilityUnavailableError, - AIDialectAdapter, - AIExecutionRequirement, - AIFeatureUnavailableError, - AIInputUnavailableError, - AIModelDisabledError, - AIModelNotFoundError, - AIOutputContractError, - AIProviderDisabledError, - AIProviderNotFoundError, - DuplicateAIDialectRegistrationError, - InvalidAIProviderConfigError, - UnknownAIDialectError, -) - - -ConfigModelT = typing.TypeVar("ConfigModelT", bound=pydantic.BaseModel) -AdapterT = typing.TypeVar("AdapterT", bound=type[AIDialectAdapter]) - - -@dataclass(frozen=True) -class _DialectRegistration: - adapter: AIDialectAdapter - adapter_type: type[AIDialectAdapter] - config_model: type[pydantic.BaseModel] - description: str - - -@dataclass(frozen=True) -class _ExecutionTarget: - model: AIModelModel - provider: AIProviderModel - adapter: AIDialectAdapter - config: pydantic.BaseModel - - -class AIManager: - """Sole domain manager for AI dialects, shared facts and execution.""" - - _DIALECTS: dict[AIDialectID, _DialectRegistration] = {} - - @classmethod - async def get_model(cls, model_id: AIModelID) -> AIModelModel | None: - """Read a model without requiring its provider/dialect to execute here.""" - async with ai_uow() as models: - return await models.get_model(model_id) - - @classmethod - async def list_models( - cls, *, limit: int | None = None, cursor: int | None = None - ) -> tuple[list[AIModelModel], int | None]: - async with ai_uow() as models: - return await models.list_models(limit=limit, cursor=cursor) - - @classmethod - def register_dialect( - cls, - dialect_id: AIDialectID, - *, - description: str, - config_model: type[ConfigModelT], - ) -> typing.Callable[[AdapterT], AdapterT]: - """Bind one adapter class to an exact ID and return it unchanged.""" - - def decorator(adapter_type: AdapterT) -> AdapterT: - existing = cls._DIALECTS.get(dialect_id) - if existing is not None: - if ( - existing.adapter_type is adapter_type - and existing.config_model is config_model - and existing.description == description - ): - return adapter_type - raise DuplicateAIDialectRegistrationError( - f"AI dialect {dialect_id!r} is already registered by " - f"{existing.adapter_type.__qualname__}" - ) - cls._DIALECTS[dialect_id] = _DialectRegistration( - adapter=adapter_type(), - adapter_type=adapter_type, - config_model=config_model, - description=description, - ) - return adapter_type - - return decorator - - @classmethod - async def sync_dialects_async(cls) -> None: - records = [] - for dialect_id, registration in cls._DIALECTS.items(): - builtin = BUILTIN_AI_DIALECTS_BY_ID.get(dialect_id) - records.append( - { - "id": dialect_id, - "description": builtin.description - if builtin is not None - else registration.description, - "config_schema": builtin.config_schema - if builtin is not None - else registration.config_model.model_json_schema(), - } - ) - async with ai_uow() as models: - await models.sync_dialects(records) - - @classmethod - def _registration(cls, dialect_id: AIDialectID) -> _DialectRegistration: - try: - return cls._DIALECTS[dialect_id] - except KeyError as error: - raise UnknownAIDialectError( - f"No local adapter implements AI dialect {dialect_id!r}" - ) from error - - @classmethod - async def _load_target_async(cls, model_id: AIModelID) -> _ExecutionTarget: - async with ai_uow() as models: - model = await models.get_model(model_id) - provider = None if model is None else await models.get_provider(model.provider) - return cls._execution_target(model_id, model, provider) - - @classmethod - def _execution_target( - cls, model_id: AIModelID, model: AIModelModel | None, provider: AIProviderModel | None - ) -> _ExecutionTarget: - if model is None: - raise AIModelNotFoundError(f"AI model {model_id} does not exist") - if not model.enabled: - raise AIModelDisabledError(f"AI model {model_id} is disabled") - if provider is None: - raise AIProviderNotFoundError( - f"AI model {model_id} references missing provider {model.provider}" - ) - if not provider.enabled: - raise AIProviderDisabledError(f"AI provider {provider.id} is disabled") - registration = cls._registration(provider.dialect) - try: - config = registration.config_model.model_validate(provider.config) - except pydantic.ValidationError as error: - raise InvalidAIProviderConfigError( - f"AI provider {provider.id} does not satisfy dialect {provider.dialect!r}" - ) from error - return _ExecutionTarget(model, provider, registration.adapter, config) - - @staticmethod - def _capability( - model: AIModelModel, - capability_type: AICapabilityType, - ) -> AIModelCapability: - capability = next( - (item for item in model.capabilities if item.type == capability_type), - None, - ) - if capability is None: - raise AICapabilityUnavailableError( - f"AI model {model.id} does not declare {capability_type!r}" - ) - return capability - - @staticmethod - def _require_modalities( - model: AIModelModel, - capability: AIModelCapability, - *, - input_: str, - output: str, - ) -> None: - if ( - input_ not in capability.input_modalities - or output not in capability.output_modalities - ): - raise AICapabilityUnavailableError( - f"AI model {model.id} {capability.type!r} does not support {input_!r} -> {output!r}" - ) - - @classmethod - async def can_execute( - cls, - model: AIModelID, - requirement: AIExecutionRequirement, - ) -> bool: - """Return static peer-local eligibility without probing a remote provider.""" - try: - target = await cls._load_target_async(model) - capability = cls._capability(target.model, requirement.capability) - except ( - AIModelNotFoundError, - AIModelDisabledError, - AIProviderNotFoundError, - AIProviderDisabledError, - UnknownAIDialectError, - InvalidAIProviderConfigError, - AICapabilityUnavailableError, - ): - return False - requested_inputs = requirement.input_modalities - requested_outputs = requirement.output_modalities - requested_features = requirement.features - if not requested_inputs <= set(capability.input_modalities): - return False - if not requested_outputs <= set(capability.output_modalities): - return False - if not requested_features <= set(capability.features): - return False - if any( - not target.adapter.supports_input_modality(requirement.capability, modality) - for modality in requested_inputs - ): - return False - if any( - not target.adapter.supports_feature(requirement.capability, feature) - for feature in requested_features - ): - return False - return requirement.tool_choice is None or target.adapter.supports_tool_choice( - requirement.tool_choice - ) - - @classmethod - async def embed( - cls, - model: AIModelID, - inputs: Sequence[str], - dimensions: int, - ) -> tuple[Vector, ...]: - """Embed one ordered non-empty text batch through the selected model.""" - with operation( - "ai.embed", - kind=SpanKind.CLIENT, - attributes={ - "gen_ai.operation.name": "embeddings", - "inkcre.ai.model.id": model, - "inkcre.ai.usage.input.source": "unavailable", - "inkcre.ai.usage.output.source": "unavailable", - }, - ) as observation: - span = observation.span - if not inputs: - raise ValueError("embedding inputs must not be empty") - if dimensions <= 0: - raise ValueError("embedding dimensions must be positive") - target = await cls._load_target_async(model) - span.set_attributes( - model_attributes( - target.model.native_model_id, target.provider.dialect, target.model.provider - ) - ) - capability = cls._capability(target.model, "embedding") - cls._require_modalities( - target.model, - capability, - input_="text", - output="vector", - ) - vectors = await target.adapter.embed( - target.config, - target.model.native_model_id, - tuple(inputs), - dimensions, - ) - if len(vectors) != len(inputs): - raise AIOutputContractError( - f"Embedding result count {len(vectors)} does not match input count {len(inputs)}" - ) - for vector in vectors: - if len(vector) != dimensions: - raise AIOutputContractError( - f"Embedding dimension {len(vector)} does not match requested {dimensions}" - ) - if not vector or not all(math.isfinite(value) for value in vector): - raise AIOutputContractError("Embedding vectors must be finite and non-empty") - if not any(value != 0 for value in vector): - raise AIOutputContractError("Embedding vectors must be non-zero") - return vectors - - @classmethod - async def chat( - cls, - model: AIModelID, - messages: Sequence[Message], - tools: Sequence[FunctionTool] = (), - tool_choice: ToolChoice | None = None, - ) -> AssistantMessage: - """Execute one provider-neutral chat model call without owning history.""" - with operation( - "ai.chat", - kind=SpanKind.CLIENT, - attributes={ - "gen_ai.operation.name": "chat", - "inkcre.ai.model.id": model, - "inkcre.ai.usage.input.source": "unavailable", - "inkcre.ai.usage.output.source": "unavailable", - }, - ) as observation: - span = observation.span - history = validate_message_history(messages) - target = await cls._load_target_async(model) - span.set_attributes( - model_attributes( - target.model.native_model_id, target.provider.dialect, target.model.provider - ) - ) - capability = cls._capability(target.model, "chat") - # System instructions, Tool schemas/results and Assistant history all travel - # through the textual chat channel even when the latest User turn is media-only. - input_modalities: set[str] = {"text"} - for message in history: - if not isinstance(message, UserMessage): - continue - for part in message.content: - if isinstance(part, TextContentPart): - input_modalities.add("text") - elif isinstance(part, ImageContentPart): - input_modalities.add("image") - elif isinstance(part, AudioContentPart): - input_modalities.add("audio") - elif isinstance(part, VideoContentPart): - input_modalities.add("video") - for modality in sorted(input_modalities): - cls._require_modalities( - target.model, - capability, - input_=modality, - output="text", - ) - if not target.adapter.supports_input_modality("chat", modality): - raise AIInputUnavailableError( - f"AI dialect {target.provider.dialect!r} cannot convey " - f"chat modality {modality!r}" - ) - - tool_ids = tuple(tool.id for tool in tools) - if len(tool_ids) != len(set(tool_ids)): - raise ValueError("Tool IDs must be unique") - requires_tool_calling = bool(tools) or tool_choice is not None - if requires_tool_calling: - if "tool_calling" not in capability.features or not target.adapter.supports_feature( - "chat", "tool_calling" - ): - raise AIFeatureUnavailableError( - f"AI model {model} and dialect {target.provider.dialect!r} do not jointly " - "support tool_calling" - ) - if tool_choice is not None: - if not tools: - raise ValueError("tool_choice requires at least one Tool") - if not target.adapter.supports_tool_choice(tool_choice): - raise AIFeatureUnavailableError( - f"AI dialect {target.provider.dialect!r} cannot represent tool_choice" - ) - - return await target.adapter.chat( - target.config, - target.model.native_model_id, - history, - tuple(tools), - tool_choice, - ) +if TYPE_CHECKING: + from inkcre_core.business.ai.main import * diff --git a/app/business/ai/telemetry.py b/app/business/ai/telemetry.py index 00431aae..000c69ca 100644 --- a/app/business/ai/telemetry.py +++ b/app/business/ai/telemetry.py @@ -1,86 +1,10 @@ -"""Content-free mapping for the pinned GenAI semantic conventions. +"""Legacy import; use inkcre_core for new consumers.""" -Keys follow semantic-conventions-genai e07f4ebacb08f56db8c4c882d117720333fbca04. -Provider responses are untrusted: copy numeric usage and known finish reasons only. -""" +from typing import TYPE_CHECKING +from importlib import import_module +import sys -from collections.abc import Sequence -import re +sys.modules[__name__] = import_module("inkcre_core.business.ai.telemetry") -from openai.types.completion_usage import CompletionUsage -from openai.types.create_embedding_response import Usage as EmbeddingUsage -from opentelemetry import trace -from opentelemetry.util.types import AttributeValue - -from libs.obsrv.telemetry import is_enabled, record_ai_usage - - -_IDENTIFIER = re.compile(r"[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}\Z") -_FINISH_REASONS = frozenset( - {"stop", "length", "tool_calls", "content_filter", "function_call"} -) - - -def model_attributes( - native_model_id: str, dialect: str, provider_id: int -) -> dict[str, AttributeValue]: - attributes: dict[str, AttributeValue] = { - "inkcre.ai.provider.id": provider_id, - "inkcre.ai.dialect": dialect if _IDENTIFIER.fullmatch(dialect) else "other", - "gen_ai.provider.name": { - "core.openai-compatible.v1": "openai", - "core.alibaba-model-studio.v1": "alibaba", - }.get(dialect, "other"), - } - # This is the configured model identifier, never the provider's arbitrary echo. - if _IDENTIFIER.fullmatch(native_model_id): - attributes["gen_ai.request.model"] = native_model_id - return attributes - - -def record_response_usage( - operation: str, - usage: CompletionUsage | EmbeddingUsage | None, - finish_reasons: Sequence[str] = (), -) -> None: - """Record one response's totals once; absent or invalid totals stay unknown.""" - if not is_enabled(): - return - span = trace.get_current_span() - input_tokens = getattr(usage, "prompt_tokens", None) - output_tokens = getattr(usage, "completion_tokens", None) - input_tokens = ( - input_tokens if type(input_tokens) is int and 0 <= input_tokens < 2**63 else None - ) - output_tokens = ( - output_tokens if type(output_tokens) is int and 0 <= output_tokens < 2**63 else None - ) - for direction, value in (("input", input_tokens), ("output", output_tokens)): - span.set_attribute( - f"inkcre.ai.usage.{direction}.source", - "provider" if value is not None else "unavailable", - ) - if value is not None: - span.set_attribute(f"gen_ai.usage.{direction}_tokens", value) - if isinstance(usage, CompletionUsage): - for key, value in ( - ( - "gen_ai.usage.cache_read.input_tokens", - getattr(usage.prompt_tokens_details, "cached_tokens", None), - ), - ( - "gen_ai.usage.reasoning.output_tokens", - getattr(usage.completion_tokens_details, "reasoning_tokens", None), - ), - ): - if type(value) is int and 0 <= value < 2**63: - span.set_attribute(key, value) - if finish_reasons: - span.set_attribute( - "gen_ai.response.finish_reasons", - tuple( - reason if isinstance(reason, str) and reason in _FINISH_REASONS else "other" - for reason in finish_reasons[:16] - ), - ) - record_ai_usage(operation, input_tokens=input_tokens, output_tokens=output_tokens) +if TYPE_CHECKING: + from inkcre_core.business.ai.telemetry import * diff --git a/app/business/cron.py b/app/business/cron.py index 3bb7765f..64f13f2d 100644 --- a/app/business/cron.py +++ b/app/business/cron.py @@ -1,231 +1,10 @@ -"""Application-owned materialization of deployment-wide Cron occurrences.""" +"""Legacy import; use inkcre_core for new consumers.""" -from __future__ import annotations +from typing import TYPE_CHECKING +from importlib import import_module +import sys -import datetime -import typing -from zoneinfo import ZoneInfo, ZoneInfoNotFoundError +sys.modules[__name__] = import_module("inkcre_core.business.cron") -import croniter # pyrefly: ignore[untyped-import] -import pydantic - -from app.business.deployment_config import DeploymentConfigManager, DeploymentConfigService -from app.business.job import JobManager -from app.persistence.cron.uow import cron_uow -from app.schemas.cron import CronForm, CronID, CronModel, CronUpdateForm -from app.schemas.job import JobModel, JobStatus -from libs.obsrv.main import get_logger -from libs.obsrv.telemetry import operation, emit_event -from app.validation import input_path - - -CRON_CONFIG_KEY = "core.cron" -CRON_CONFIG_SCHEMA_ID = "core.cron.config.v1" -LOGGER = get_logger().getChild(__name__) - - -class CronNotFoundError(LookupError): - """A Cron template does not exist.""" - - -class InvalidCronScheduleError(ValueError): - """A submitted schedule is not a five-field UNIX Cron expression.""" - - -class CronDeploymentConfig(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid") - - timezone: str = "UTC" - - @pydantic.field_validator("timezone") - @classmethod - def validate_timezone(cls, value: str) -> str: - try: - ZoneInfo(value) - except ZoneInfoNotFoundError as error: - raise ValueError("timezone must be an installed IANA timezone") from error - return value - - -DeploymentConfigManager.register_schema( - CRON_CONFIG_SCHEMA_ID, CronDeploymentConfig, keys=(CRON_CONFIG_KEY,) -) - - -class InvalidCronTemplateError(ValueError): - """A failed template must leave its complete transaction before discovery continues.""" - - -class CronManager: - """Evaluate only the current minute and atomically create typed Jobs.""" - - @classmethod - async def _timezone(cls) -> ZoneInfo: - config = await DeploymentConfigService.get(CRON_CONFIG_KEY) - if config is None: - return ZoneInfo("UTC") - return ZoneInfo(typing.cast(CronDeploymentConfig, config).timezone) - - @classmethod - def _matches(cls, schedule: str, now: datetime.datetime) -> bool: - return croniter.croniter.match(schedule, now) - - @classmethod - async def check(cls) -> int: - """Materialize each matching occurrence in one independent transaction.""" - async with cron_uow() as uow: - cron_ids = await uow.crons.enabled_ids() - timezone = await cls._timezone() - created = 0 - for cron_id in cron_ids: - if cron_id is None: - continue - try: - created += await cls._materialize(cron_id, timezone) - except InvalidCronTemplateError: - LOGGER.exception("Cron Job template is invalid", extra={"cron_id": cron_id}) - return created - - @classmethod - async def _materialize(cls, cron_id: CronID, timezone: ZoneInfo) -> int: - with operation("cron.materialize", attributes={"inkcre.cron.id": cron_id}): - return await cls._materialize_occurrence(cron_id, timezone) - - @classmethod - async def _materialize_occurrence(cls, cron_id: CronID, timezone: ZoneInfo) -> int: - async with cron_uow() as uow: - cron = await uow.crons.get(cron_id, lock=True, skip_locked=True) - if cron is None or not cron.enabled: - return 0 - database_now = await uow.crons.database_now() - occurrence = database_now.astimezone(datetime.UTC).replace(second=0, microsecond=0) - local_minute = database_now.astimezone(timezone).replace(second=0, microsecond=0) - try: - matches = cls._matches(cron.schedule, local_minute) - except (ValueError, KeyError): - LOGGER.exception("Invalid persisted Cron schedule", extra={"cron_id": cron.id}) - return 0 - if not matches or cron.last_scheduled_for == occurrence: - return 0 - if cron.last_job is not None: - last_job = await uow.jobs.get(cron.last_job) - if last_job is not None and last_job.status in { - JobStatus.PENDING, - JobStatus.RUNNING, - }: - return 0 - try: - job = await JobManager.create_in_uow( - cron.job_type, cron.job_parameters, cron.job_timeout_seconds, uow=uow - ) - except Exception as error: - raise InvalidCronTemplateError(f"Invalid Cron {cron.id} template") from error - cron.last_job = job.id - cron.last_scheduled_for = occurrence - await uow.crons.save(cron) - emit_event("job.submitted", {"inkcre.job.id": job.id or 0, "inkcre.cron.id": cron_id}) - return 1 - - @classmethod - async def run_now(cls, cron_id: CronID) -> JobModel: - """Create a Job from one Cron template without changing Cron progress.""" - async with cron_uow() as uow: - cron = await uow.crons.get(cron_id) - if cron is None: - raise CronNotFoundError(f"Cron {cron_id} does not exist") - job = await JobManager.create_in_uow( - cron.job_type, - cron.job_parameters, - cron.job_timeout_seconds, - uow=uow, - ) - emit_event("job.submitted", {"inkcre.job.id": job.id or 0, "inkcre.cron.id": cron_id}) - return job - - @classmethod - async def create(cls, form: CronForm) -> CronModel: - """Validate and create one Cron template.""" - if len(form.schedule.split()) != 5 or not croniter.croniter.is_valid(form.schedule): - raise InvalidCronScheduleError( - "Cron schedule must be a valid five-field UNIX expression" - ) - async with cron_uow() as uow: - cron = CronModel(**form.model_dump()) - with input_path("job_parameters"): - cron.job_parameters = await JobManager.normalize_parameters( - form.job_type, form.job_parameters, uow - ) - await uow.crons.save(cron) - return cron - - @classmethod - async def update(cls, cron_id: CronID, form: CronForm) -> CronModel: - """Validate and replace the editable fields of one Cron template.""" - if len(form.schedule.split()) != 5 or not croniter.croniter.is_valid(form.schedule): - raise InvalidCronScheduleError( - "Cron schedule must be a valid five-field UNIX expression" - ) - async with cron_uow() as uow: - cron = await uow.crons.get(cron_id) - if cron is None: - raise CronNotFoundError(f"Cron {cron_id} does not exist") - cron.schedule = form.schedule - cron.enabled = form.enabled - cron.job_type = form.job_type - with input_path("job_parameters"): - cron.job_parameters = await JobManager.normalize_parameters( - form.job_type, form.job_parameters, uow - ) - cron.job_timeout_seconds = form.job_timeout_seconds - await uow.crons.save(cron) - return cron - - @classmethod - async def get(cls, cron_id: CronID) -> CronModel | None: - async with cron_uow() as uow: - return await uow.crons.get(cron_id) - - @classmethod - async def list_crons( - cls, *, limit: int | None = None, cursor: CronID | None = None - ) -> tuple[list[CronModel], CronID | None]: - async with cron_uow() as uow: - return await uow.crons.list(limit=limit, cursor=cursor) - - @classmethod - async def patch(cls, cron_id: CronID, form: CronUpdateForm) -> CronModel: - async with cron_uow() as uow: - cron = await uow.crons.get(cron_id, lock=True) - if cron is None: - raise CronNotFoundError(f"Cron {cron_id} does not exist") - changes = form.model_dump(exclude_unset=True) - candidate = CronForm.model_validate( - { - **{field: getattr(cron, field) for field in CronForm.model_fields}, - **changes, - } - ) - if len(candidate.schedule.split()) != 5 or not croniter.croniter.is_valid( - candidate.schedule - ): - raise InvalidCronScheduleError( - "Cron schedule must be a valid five-field UNIX expression" - ) - with input_path("job_parameters"): - parameters = await JobManager.normalize_parameters( - candidate.job_type, candidate.job_parameters, uow - ) - for field in changes: - setattr(cron, field, getattr(candidate, field)) - cron.job_parameters = parameters - await uow.crons.save(cron) - return cron - - @classmethod - async def delete(cls, cron_id: CronID) -> bool: - async with cron_uow() as uow: - cron = await uow.crons.get(cron_id) - if cron is None: - return False - await uow.crons.delete(cron) - return True +if TYPE_CHECKING: + from inkcre_core.business.cron import * diff --git a/app/business/deployment_config.py b/app/business/deployment_config.py index cda9963c..db921256 100644 --- a/app/business/deployment_config.py +++ b/app/business/deployment_config.py @@ -1,186 +1,10 @@ -"""Deployment-scoped configuration registry and persistence.""" +"""Legacy import; use inkcre_core for new consumers.""" -import typing -import dataclasses +from typing import TYPE_CHECKING +from importlib import import_module +import sys -import pydantic +sys.modules[__name__] = import_module("inkcre_core.business.deployment_config") -from app.persistence.deployment_config.uow import configuration_transaction -from app.configuration import ConfigContract -from app.schemas.deployment_config import ( - DeploymentConfigKey, - DeploymentConfigModel, - DeploymentConfigSchemaID, - DeploymentConfigView, -) - - -class UnknownDeploymentConfigSchemaError(LookupError): - """The current peer has no validator for an exact persisted schema ID.""" - - -class DeploymentConfigSchemaCollisionError(ValueError): - """Two different local models claimed the same exact schema ID.""" - - -class DeploymentConfigNotFoundError(LookupError): - """A patch addressed a deployment config that does not exist.""" - - -@dataclasses.dataclass(frozen=True) -class _SchemaRegistration: - contract: ConfigContract - keys: tuple[str, ...] - - -class DeploymentConfigManager: - """Own exact schema registration and the shared ``configs`` relation.""" - - _contracts: dict[DeploymentConfigSchemaID, _SchemaRegistration] = {} - - @classmethod - def register_schema( - cls, - schema_id: DeploymentConfigSchemaID, - model: type[pydantic.BaseModel], - *, - keys: tuple[str, ...] = (), - ) -> None: - """Register one exact schema ID idempotently for the same model.""" - existing = cls._contracts.get(schema_id) - if existing is not None: - if existing.contract.model is model: - cls._contracts[schema_id] = _SchemaRegistration( - existing.contract, tuple(sorted(set(existing.keys) | set(keys))) - ) - return - raise DeploymentConfigSchemaCollisionError( - f"Deployment config schema {schema_id!r} is already registered " - f"by {existing.contract.model.__qualname__}" - ) - cls._contracts[schema_id] = _SchemaRegistration(ConfigContract(model), keys) - - @classmethod - def _contract( - cls, - schema_id: DeploymentConfigSchemaID, - ) -> ConfigContract: - try: - return cls._contracts[schema_id].contract - except KeyError as error: - raise UnknownDeploymentConfigSchemaError( - f"Unknown deployment config schema: {schema_id}" - ) from error - - @classmethod - def _restore_record( - cls, - record: DeploymentConfigModel, - ) -> pydantic.BaseModel: - return cls._contract(record.schema_id).validate(record.value) - - @classmethod - def _view(cls, record: DeploymentConfigModel) -> DeploymentConfigView: - return DeploymentConfigView( - key=record.key, - schema=record.schema_id, - value=record.value, - created_at=record.created_at, - updated_at=record.updated_at, - ) - - @classmethod - def get_schema(cls, schema_id: str, *, include_schema: bool = True) -> dict: - contract = cls._contract(schema_id) - entry = cls._contracts[schema_id] - result: dict[str, typing.Any] = { - "id": schema_id, - "keys": entry.keys, - "description": contract.model.__doc__ or "", - } - if include_schema: - result["input_schema"] = contract.json_schema() - return result - - @classmethod - def list_schemas( - cls, *, limit: int | None = None, cursor: str | None = None - ) -> tuple[list[dict], str | None]: - ids = sorted(key for key in cls._contracts if cursor is None or key > cursor) - more = limit is not None and len(ids) > limit - ids = ids[:limit] - return [cls.get_schema(key, include_schema=False) for key in ids], ids[ - -1 - ] if more else None - - -class DeploymentConfigService: - """Deployment configuration use cases over the process-local schema registry.""" - - @staticmethod - async def get(key: DeploymentConfigKey) -> pydantic.BaseModel | None: - async with configuration_transaction() as configs: - record = await configs.get(key) - return None if record is None else DeploymentConfigManager._restore_record(record) - - @staticmethod - async def read(key: DeploymentConfigKey) -> DeploymentConfigView | None: - async with configuration_transaction() as configs: - record = await configs.get(key) - return None if record is None else DeploymentConfigManager._view(record) - - @staticmethod - async def replace( - key: DeploymentConfigKey, - schema_id: DeploymentConfigSchemaID, - complete_value: dict[str, typing.Any], - ) -> DeploymentConfigView: - result, _ = await DeploymentConfigService.replace_with_status( - key, schema_id, complete_value - ) - return result - - @staticmethod - async def replace_with_status( - key: DeploymentConfigKey, - schema_id: DeploymentConfigSchemaID, - complete_value: dict[str, typing.Any], - ) -> tuple[DeploymentConfigView, bool]: - normalized = DeploymentConfigManager._contract(schema_id).normalize(complete_value) - async with configuration_transaction() as configs: - record, created = await configs.replace(key, schema_id, normalized) - result = DeploymentConfigManager._view(record) - return result, created - - @staticmethod - async def list_configs( - *, limit: int | None = None, cursor: str | None = None - ) -> tuple[list[DeploymentConfigView], str | None]: - async with configuration_transaction() as configs: - rows, next_cursor = await configs.list(limit=limit, cursor=cursor) - return [DeploymentConfigManager._view(row) for row in rows], next_cursor - - @staticmethod - async def delete(key: DeploymentConfigKey) -> bool: - async with configuration_transaction() as configs: - record = await configs.get(key) - if record is None: - return False - await configs.delete(record) - return True - - @staticmethod - async def patch( - key: DeploymentConfigKey, partial_value: dict[str, typing.Any] - ) -> DeploymentConfigView: - async with configuration_transaction() as configs: - record = await configs.get(key, for_update=True) - if record is None: - raise DeploymentConfigNotFoundError(f"Deployment config {key!r} does not exist") - contract = DeploymentConfigManager._contract(record.schema_id) - record.value = contract.prepare_patch(record.value, partial_value).model_dump( - mode="json" - ) - await configs.save(record) - result = DeploymentConfigManager._view(record) - return result +if TYPE_CHECKING: + from inkcre_core.business.deployment_config import * diff --git a/app/business/extension/AGENTS.md b/app/business/extension/AGENTS.md index 69fd7c97..3005bf20 100644 --- a/app/business/extension/AGENTS.md +++ b/app/business/extension/AGENTS.md @@ -1,7 +1,7 @@ # extension/ Local Guide -本目录实现唯一的 Core Extension Host。修改前同时核对 `main.py`、`state.py`、 -`release.py`、`distribution.py`、`app/schemas/extension/main.py` 与 `run.py`。 +本目录实现服务的 Extension Host 装配。修改前同时核对 `main.py`、`state.py`、 +`release.py`、`distribution.py`、`inkcre_core/schemas/extension/main.py` 与 `run.py`。 ## 稳定边界 @@ -35,7 +35,7 @@ ## 权限和持久化 `state.py` 拥有异步持久化用例与 shared state/config 规则；SQL 和 session-bound repository 位于 -`app/persistence/extension/`，UoW 工厂使用原生异步事务。`extensions.state` 是 deployment-wide Extension-produced state； +`inkcre_core/persistence/extension/`，UoW 工厂使用原生异步事务。`extensions.state` 是 deployment-wide Extension-produced state； `extension_peer_participations` 保存 Peer-local selection、enabled intent 与精确版本 config schema。SQLModel 不应泄露成 Host 的稳定接口。 Extension authoring 与 wheel metadata 规则见 `extensions/AGENTS.md`。 diff --git a/app/business/extension/config.py b/app/business/extension/config.py index 1e95d7f3..778969cb 100644 --- a/app/business/extension/config.py +++ b/app/business/extension/config.py @@ -5,8 +5,11 @@ import pydantic -from app.business.deployment_config import DeploymentConfigManager, DeploymentConfigService -from app.business.peer import PeerManager +from inkcre_core.business.deployment_config import ( + DeploymentConfigManager, + DeploymentConfigService, +) +from inkcre_core.business.peer import PeerManager from app.settings import settings diff --git a/app/business/extension/main.py b/app/business/extension/main.py index b23bab72..631c1c09 100644 --- a/app/business/extension/main.py +++ b/app/business/extension/main.py @@ -18,8 +18,8 @@ from packaging.utils import canonicalize_name import sqlmodel -from app.business.peer import PeerManager -from app.schemas.extension import ( +from inkcre_core.business.peer import PeerManager +from inkcre_core.schemas.extension import ( DisableExtensionCommand, EnableExtensionCommand, ExtensionPeerStatus, @@ -30,11 +30,11 @@ SelectExtensionCommand, UnselectExtensionCommand, ) -from app.schemas.peer import PeerProtocolRequest, PeerProtocolResponse, PeerRef +from inkcre_core.schemas.peer import PeerProtocolRequest, PeerProtocolResponse, PeerRef from app.settings import settings from app.version import CORE_VERSION from app.runtime import request_process_restart -from libs.obsrv.main import get_logger +from inkcre_core.libs.obsrv.main import get_logger from .distribution import ( AcquiredDistribution, diff --git a/app/business/extension/state.py b/app/business/extension/state.py index 8f37aa72..187034ce 100644 --- a/app/business/extension/state.py +++ b/app/business/extension/state.py @@ -8,8 +8,8 @@ import pydantic -from app.persistence.extension.uow import extension_uow -from app.schemas.extension import ExtensionModel, ExtensionPeerParticipationModel +from inkcre_core.persistence.extension.uow import extension_uow +from inkcre_core.schemas.extension import ExtensionModel, ExtensionPeerParticipationModel from .errors import ExtensionNotInstalledError, ExtensionStateConflictError diff --git a/app/business/graph_navigation_retrieval/__init__.py b/app/business/graph_navigation_retrieval/__init__.py index 0b10b714..c4601a7f 100644 --- a/app/business/graph_navigation_retrieval/__init__.py +++ b/app/business/graph_navigation_retrieval/__init__.py @@ -1,3 +1,3 @@ -from .main import GraphNavigationRetrievalManager +"""Legacy import; use inkcre_core for new consumers.""" -__all__ = ["GraphNavigationRetrievalManager"] +from inkcre_core.business.graph_navigation_retrieval import * diff --git a/app/business/graph_navigation_retrieval/main.py b/app/business/graph_navigation_retrieval/main.py index 92b2cccf..249957a2 100644 --- a/app/business/graph_navigation_retrieval/main.py +++ b/app/business/graph_navigation_retrieval/main.py @@ -1,476 +1,12 @@ -"""Bounded, presentation-neutral navigation over persisted graph authority.""" +"""Legacy import; use inkcre_core for new consumers.""" -from collections import deque -import typing +from typing import TYPE_CHECKING +from importlib import import_module +import sys - -from app.persistence.info_base.uow import GraphUnitOfWork, graph_uow -from app.schemas.graph_navigation_retrieval import ( - DEFAULT_NEIGHBORHOOD_LIMIT, - MAX_NEIGHBORHOOD_LIMIT, - DEFAULT_MAX_HOPS, - MAX_MAX_HOPS, - DEFAULT_MAX_EXPLORED_BLOCKS, - MAX_MAX_EXPLORED_BLOCKS, - DEFAULT_MAX_EXPLORED_RELATIONS, - BlockNeighborhood, - ConnectedComponentsResult, - ConnectedSeedComponent, - GraphDirection, - GraphModel, - PathFound, - PathLimitReached, - PathNotFound, - PathResult, - RelationNeighborhood, +sys.modules[__name__] = import_module( + "inkcre_core.business.graph_navigation_retrieval.main" ) -from app.schemas.info_base.block import BlockID, BlockModel -from app.schemas.info_base.relation import RelationID, RelationModel - - -FRONTIER_QUERY_SIZE = 200 - - -class GraphNavigationRetrievalManager: - """Own bounded graph-navigation queries over persisted entities.""" - - @classmethod - async def get_random_block( - cls, - ) -> BlockModel | None: - async with graph_uow() as uow: - return await uow.blocks.get_random() - - @classmethod - async def get_block_neighborhood( - cls, - focal_block: BlockID, - *, - direction: GraphDirection = "both", - contents: typing.Collection[str] = (), - limit: int = DEFAULT_NEIGHBORHOOD_LIMIT, - cursor: RelationID | None = None, - ) -> BlockNeighborhood | None: - if not 1 <= limit <= MAX_NEIGHBORHOOD_LIMIT: - raise ValueError(f"limit must be between 1 and {MAX_NEIGHBORHOOD_LIMIT}") - async with graph_uow() as uow: - focal = await uow.blocks.get(focal_block) - if focal is None: - return None - - requested = limit + 1 - branches: list[tuple[RelationModel, ...]] = [] - if direction in ("out", "both"): - branches.append( - await uow.relations.get_endpoint_page( - (focal_block,), - endpoint="from", - contents=contents, - cursor=cursor, - limit=requested, - ) - ) - if direction in ("in", "both"): - branches.append( - await uow.relations.get_endpoint_page( - (focal_block,), endpoint="to", contents=contents, cursor=cursor, limit=requested - ) - ) - by_id = { - relation.id: relation - for branch in branches - for relation in branch - if relation.id is not None - } - ordered = sorted( - by_id.values(), - key=lambda relation: typing.cast(int, relation.id), - reverse=True, - ) - page = ordered[:requested] - has_more = len(page) > limit - page = page[:limit] - endpoint_ids = {focal_block} - for relation in page: - endpoint_ids.update((relation.from_, relation.to_)) - blocks = await uow.blocks.get_many(endpoint_ids) - existing_ids = {block.id for block in blocks} - closed_relations = tuple( - relation - for relation in page - if relation.from_ in existing_ids and relation.to_ in existing_ids - ) - next_cursor = typing.cast(RelationID, page[-1].id) if has_more and page else None - return BlockNeighborhood( - focal_block=focal_block, - graph=GraphModel(blocks=blocks, relations=closed_relations), - next_cursor=next_cursor, - ) - - @classmethod - async def get_relation_neighborhood( - cls, - focal_relation: RelationID, - ) -> RelationNeighborhood | None: - async with graph_uow() as uow: - relation = await uow.relations.get_by_id(focal_relation) - if relation is None: - return None - blocks = await uow.blocks.get_many((relation.from_, relation.to_)) - if {block.id for block in blocks} != {relation.from_, relation.to_}: - return None - return RelationNeighborhood( - focal_relation=focal_relation, - graph=GraphModel(blocks=blocks, relations=(relation,)), - ) - - @classmethod - async def get_connected_components( - cls, - seed_block_ids: typing.Collection[BlockID], - *, - contents: typing.Collection[str], - max_explored_blocks: int = DEFAULT_MAX_EXPLORED_BLOCKS, - max_explored_relations: int = DEFAULT_MAX_EXPLORED_RELATIONS, - ) -> ConnectedComponentsResult: - """Partition existing seeds by bounded undirected exact-content reachability.""" - seeds = tuple(dict.fromkeys(seed_block_ids)) - relation_contents = tuple(dict.fromkeys(contents)) - if not relation_contents: - raise ValueError("contents must not be empty") - if max_explored_blocks < 1 or max_explored_relations < 1: - raise ValueError("exploration bounds must be positive") - if len(seeds) > max_explored_blocks: - raise ValueError("seed blocks exceed max_explored_blocks") - async with graph_uow() as uow: - existing_blocks = await uow.blocks.get_many(seeds) - existing_seed_ids = {block.id for block in existing_blocks if block.id is not None} - missing = tuple(seed for seed in seeds if seed not in existing_seed_ids) - assigned_seeds: set[BlockID] = set() - explored_blocks = set(existing_seed_ids) - seen_relations: set[RelationID] = set() - explored_relation_count = 0 - proof_relations: dict[RelationID, RelationModel] = {} - components: list[ConnectedSeedComponent] = [] - truncated = False - - for seed in seeds: - if seed not in existing_seed_ids or seed in assigned_seeds: - continue - if truncated: - components.append( - ConnectedSeedComponent(seed_block_ids=(seed,), member_block_ids=(seed,)) - ) - assigned_seeds.add(seed) - continue - - members = {seed} - frontier = deque((seed,)) - while frontier and not truncated: - current = frontier.popleft() - for endpoint in ("from", "to"): - cursor: RelationID | None = None - while not truncated: - remaining = max_explored_relations - explored_relation_count - if remaining == 0: - truncated = True - break - requested = min(FRONTIER_QUERY_SIZE, remaining + 1) - page = await uow.relations.get_endpoint_page( - (current,), - endpoint=typing.cast(typing.Literal["from", "to"], endpoint), - contents=relation_contents, - cursor=cursor, - limit=requested, - ) - if len(page) > remaining: - page = page[:remaining] - truncated = True - explored_relation_count += len(page) - for relation in page: - if relation.id is None or relation.id in seen_relations: - continue - relation_id = relation.id - seen_relations.add(relation_id) - neighbor = relation.to_ if relation.from_ == current else relation.from_ - if neighbor in members: - continue - if ( - neighbor not in explored_blocks - and len(explored_blocks) >= max_explored_blocks - ): - truncated = True - break - members.add(neighbor) - explored_blocks.add(neighbor) - frontier.append(neighbor) - proof_relations[relation_id] = relation - if truncated or len(page) < requested: - break - cursor = typing.cast(RelationID, page[-1].id) - - component_seeds = tuple(seed_id for seed_id in seeds if seed_id in members) - assigned_seeds.update(component_seeds) - components.append( - ConnectedSeedComponent( - seed_block_ids=component_seeds, - member_block_ids=tuple(sorted(members)), - ) - ) - - proof_blocks = await uow.blocks.get_many(explored_blocks) - return ConnectedComponentsResult( - components=tuple(components), - proof_graph=GraphModel( - blocks=proof_blocks, - relations=tuple(proof_relations.values()), - ), - missing_seed_block_ids=missing, - truncated=truncated, - ) - - @classmethod - async def find_path( # noqa: PLR0913 - cls, - from_block: BlockID, - to_block: BlockID, - *, - direction: GraphDirection = "both", - contents: typing.Collection[str] = (), - max_hops: int = DEFAULT_MAX_HOPS, - max_explored_blocks: int = DEFAULT_MAX_EXPLORED_BLOCKS, - ) -> PathResult: - if not 0 <= max_hops <= MAX_MAX_HOPS: - raise ValueError(f"max_hops must be between 0 and {MAX_MAX_HOPS}") - if not 1 <= max_explored_blocks <= MAX_MAX_EXPLORED_BLOCKS: - raise ValueError( - f"max_explored_blocks must be between 1 and {MAX_MAX_EXPLORED_BLOCKS}" - ) - async with graph_uow() as uow: - endpoints = await uow.blocks.get_many((from_block, to_block)) - if {block.id for block in endpoints} != {from_block, to_block}: - return PathNotFound() - if from_block == to_block: - return PathFound( - graph=GraphModel(blocks=endpoints, relations=()), - block_path=(from_block,), - relation_path=(), - ) - - forward_parents: dict[BlockID, tuple[BlockID, RelationID] | None] = {from_block: None} - backward_next: dict[BlockID, tuple[BlockID, RelationID] | None] = {to_block: None} - forward_depths = {from_block: 0} - backward_depths = {to_block: 0} - forward_frontier = {from_block} - backward_frontier = {to_block} - traversed_relations: dict[RelationID, RelationModel] = {} - - while forward_frontier and backward_frontier: - forward_level = forward_depths[next(iter(forward_frontier))] - backward_level = backward_depths[next(iter(backward_frontier))] - if forward_level + backward_level >= max_hops: - return PathLimitReached() - - expand_forward = len(forward_frontier) <= len(backward_frontier) - if expand_forward: - next_frontier, meeting = await cls._expand_path_frontier( - forward_frontier, - direction=direction, - contents=contents, - reverse=False, - own_steps=forward_parents, - own_depths=forward_depths, - other_depths=backward_depths, - max_hops=max_hops, - traversed_relations=traversed_relations, - uow=uow, - ) - forward_frontier = next_frontier - else: - next_frontier, meeting = await cls._expand_path_frontier( - backward_frontier, - direction=direction, - contents=contents, - reverse=True, - own_steps=backward_next, - own_depths=backward_depths, - other_depths=forward_depths, - max_hops=max_hops, - traversed_relations=traversed_relations, - uow=uow, - ) - backward_frontier = next_frontier - explored = set(forward_parents) | set(backward_next) - if len(explored) > max_explored_blocks: - return PathLimitReached() - if meeting is not None: - return await cls._assemble_bidirectional_path( - from_block, - to_block, - meeting=meeting, - forward_parents=forward_parents, - backward_next=backward_next, - traversed_relations=traversed_relations, - uow=uow, - ) - return PathNotFound() - - @classmethod - async def _expand_path_frontier( # noqa: PLR0913 - cls, - frontier: set[BlockID], - *, - direction: GraphDirection, - contents: typing.Collection[str], - reverse: bool, - own_steps: dict[BlockID, tuple[BlockID, RelationID] | None], - own_depths: dict[BlockID, int], - other_depths: dict[BlockID, int], - max_hops: int, - traversed_relations: dict[RelationID, RelationModel], - uow: GraphUnitOfWork, - ) -> tuple[set[BlockID], BlockID | None]: - next_frontier: set[BlockID] = set() - meeting: BlockID | None = None - best_hops: int | None = None - frontier_items = tuple(frontier) - for chunk_start in range(0, len(frontier), FRONTIER_QUERY_SIZE): - chunk = frontier_items[chunk_start : chunk_start + FRONTIER_QUERY_SIZE] - chunk_members = set(chunk) - relations = await cls._frontier_relations( - chunk, - direction=direction, - contents=contents, - reverse=reverse, - uow=uow, - ) - for relation in relations: - relation_id = typing.cast(RelationID, relation.id) - traversed_relations[relation_id] = relation - for current, neighbor in cls._relation_steps( - relation, frontier=chunk_members, direction=direction, reverse=reverse - ): - if neighbor in own_steps: - continue - own_steps[neighbor] = (current, relation_id) - own_depths[neighbor] = own_depths[current] + 1 - next_frontier.add(neighbor) - if neighbor not in other_depths: - continue - hops = own_depths[neighbor] + other_depths[neighbor] - if hops <= max_hops and (best_hops is None or hops < best_hops): - best_hops = hops - meeting = neighbor - return next_frontier, meeting - - @staticmethod - def _relation_steps( - relation: RelationModel, - *, - frontier: set[BlockID], - direction: GraphDirection, - reverse: bool, - ) -> tuple[tuple[BlockID, BlockID], ...]: - steps: list[tuple[BlockID, BlockID]] = [] - if not reverse: - if direction in ("out", "both") and relation.from_ in frontier: - steps.append((relation.from_, relation.to_)) - if direction in ("in", "both") and relation.to_ in frontier: - steps.append((relation.to_, relation.from_)) - else: - if direction in ("out", "both") and relation.to_ in frontier: - steps.append((relation.to_, relation.from_)) - if direction in ("in", "both") and relation.from_ in frontier: - steps.append((relation.from_, relation.to_)) - return tuple(steps) - - @classmethod - async def _frontier_relations( - cls, - frontier: typing.Collection[BlockID], - *, - direction: GraphDirection, - contents: typing.Collection[str], - reverse: bool = False, - uow: GraphUnitOfWork, - ) -> tuple[RelationModel, ...]: - relations: dict[RelationID, RelationModel] = {} - needs_from = ( - direction in ("out", "both") - if not reverse - else direction - in ( - "in", - "both", - ) - ) - needs_to = ( - direction in ("in", "both") - if not reverse - else direction - in ( - "out", - "both", - ) - ) - if needs_from: - for relation in await uow.relations.get_endpoint_page( - frontier, endpoint="from", contents=contents, limit=MAX_MAX_EXPLORED_BLOCKS - ): - if relation.id is not None: - relations[relation.id] = relation - if needs_to: - for relation in await uow.relations.get_endpoint_page( - frontier, endpoint="to", contents=contents, limit=MAX_MAX_EXPLORED_BLOCKS - ): - if relation.id is not None: - relations[relation.id] = relation - return tuple(relations.values()) - @classmethod - async def _assemble_bidirectional_path( # noqa: PLR0913 - cls, - from_block: BlockID, - to_block: BlockID, - *, - meeting: BlockID, - forward_parents: dict[BlockID, tuple[BlockID, RelationID] | None], - backward_next: dict[BlockID, tuple[BlockID, RelationID] | None], - traversed_relations: dict[RelationID, RelationModel], - uow: GraphUnitOfWork, - ) -> PathResult: - block_path: list[BlockID] = [meeting] - relation_path: list[RelationID] = [] - current = meeting - while current != from_block: - parent = forward_parents[current] - if parent is None: # pragma: no cover - reconstruction invariant - raise RuntimeError("Path reconstruction lost its parent") - current, relation_id = parent - block_path.append(current) - relation_path.append(relation_id) - block_path.reverse() - relation_path.reverse() - current = meeting - while current != to_block: - next_step = backward_next[current] - if next_step is None: # pragma: no cover - reconstruction invariant - raise RuntimeError("Path reconstruction lost its next step") - current, relation_id = next_step - block_path.append(current) - relation_path.append(relation_id) - blocks = await uow.blocks.get_many(block_path) - relations = tuple( - relation - for relation_id in relation_path - if (relation := traversed_relations.get(relation_id)) is not None - ) - if {block.id for block in blocks} != set(block_path) or len(relations) != len( - relation_path - ): - return PathNotFound() - return PathFound( - graph=GraphModel(blocks=blocks, relations=relations), - block_path=tuple(block_path), - relation_path=tuple(relation_path), - ) +if TYPE_CHECKING: + from inkcre_core.business.graph_navigation_retrieval.main import * diff --git a/app/business/graph_navigation_retrieval/tools.py b/app/business/graph_navigation_retrieval/tools.py index 7c0a2eb0..16056631 100644 --- a/app/business/graph_navigation_retrieval/tools.py +++ b/app/business/graph_navigation_retrieval/tools.py @@ -1,148 +1,12 @@ -"""Agent Tool controllers for graph-navigation retrieval.""" +"""Legacy import; use inkcre_core for new consumers.""" -import typing +from typing import TYPE_CHECKING +from importlib import import_module +import sys -import pydantic - -from app.business.agent import AgentManager, ToolExecutionError -from app.business.agent.projection import project_json -from app.schemas.ai import JSONValue -from app.schemas.graph_navigation_retrieval import ( - DEFAULT_MAX_EXPLORED_BLOCKS, - DEFAULT_MAX_EXPLORED_RELATIONS, - DEFAULT_MAX_HOPS, - DEFAULT_NEIGHBORHOOD_LIMIT, - MAX_MAX_EXPLORED_BLOCKS, - MAX_MAX_HOPS, - MAX_NEIGHBORHOOD_LIMIT, - GraphDirection, +sys.modules[__name__] = import_module( + "inkcre_core.business.graph_navigation_retrieval.tools" ) -from app.schemas.info_base.block import BlockID -from app.schemas.info_base.relation import RelationID - -from .main import GraphNavigationRetrievalManager - - -GET_ENTITY_NEIGHBORHOOD_TOOL = "get_entity_neighborhood" -FIND_PATH_TOOL = "find_path" -GET_CONNECTED_COMPONENTS_TOOL = "get_connected_components" - - -class BlockNeighborhoodInput(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - entity_type: typing.Literal["block"] - entity_id: BlockID - direction: GraphDirection = "both" - contents: tuple[str, ...] = () - limit: int = pydantic.Field( - default=DEFAULT_NEIGHBORHOOD_LIMIT, ge=1, le=MAX_NEIGHBORHOOD_LIMIT - ) - cursor: RelationID | None = None - - -class RelationNeighborhoodInput(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - entity_type: typing.Literal["relation"] - entity_id: RelationID - - -class EntityNeighborhoodInput( - pydantic.RootModel[ - typing.Annotated[ - BlockNeighborhoodInput | RelationNeighborhoodInput, - pydantic.Field(discriminator="entity_type"), - ] - ] -): - model_config = pydantic.ConfigDict(json_schema_extra={"type": "object"}) - - @classmethod - def model_json_schema(cls, *args, **kwargs) -> dict[str, typing.Any]: - schema = super().model_json_schema(*args, **kwargs) - properties = BlockNeighborhoodInput.model_json_schema(*args, **kwargs)["properties"] - properties["entity_type"] = {"type": "string", "enum": ["block", "relation"]} - schema["properties"] = properties - return schema - -class FindPathInput(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - from_block_id: BlockID - to_block_id: BlockID - direction: GraphDirection = "both" - contents: tuple[str, ...] = () - max_hops: int = pydantic.Field(default=DEFAULT_MAX_HOPS, ge=0, le=MAX_MAX_HOPS) - max_explored_blocks: int = pydantic.Field( - default=DEFAULT_MAX_EXPLORED_BLOCKS, ge=1, le=MAX_MAX_EXPLORED_BLOCKS - ) - - -class ConnectedComponentsInput(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - seed_block_ids: tuple[BlockID, ...] - contents: tuple[str, ...] = pydantic.Field(min_length=1) - max_explored_blocks: int = pydantic.Field(default=DEFAULT_MAX_EXPLORED_BLOCKS, ge=1) - max_explored_relations: int = pydantic.Field(default=DEFAULT_MAX_EXPLORED_RELATIONS, ge=1) - - -@AgentManager.tool( - GET_ENTITY_NEIGHBORHOOD_TOOL, - description=( - "Read a Block's direct neighborhood or a Relation with its endpoints. " - "Null may indicate an incorrect entity type." - ), -) -async def get_entity_neighborhood(input: EntityNeighborhoodInput) -> JSONValue: - request = input.root - if request.entity_type == "block": - result = await GraphNavigationRetrievalManager.get_block_neighborhood( - request.entity_id, - direction=request.direction, - contents=request.contents, - limit=request.limit, - cursor=request.cursor, - ) - else: - result = await GraphNavigationRetrievalManager.get_relation_neighborhood( - request.entity_id - ) - return project_json(result) - - -@AgentManager.tool( - FIND_PATH_TOOL, - description="Find a bounded graph path; an exploration limit is not proof of absence.", -) -async def find_path(input: FindPathInput) -> JSONValue: - return project_json( - await GraphNavigationRetrievalManager.find_path( - input.from_block_id, - input.to_block_id, - direction=input.direction, - contents=input.contents, - max_hops=input.max_hops, - max_explored_blocks=input.max_explored_blocks, - ) - ) - - -@AgentManager.tool( - GET_CONNECTED_COMPONENTS_TOOL, - description=( - "Partition seeds by bounded undirected reachability through exact Relation contents." - ), -) -async def get_connected_components(input: ConnectedComponentsInput) -> JSONValue: - try: - result = await GraphNavigationRetrievalManager.get_connected_components( - **input.model_dump() - ) - except ValueError as error: - raise ToolExecutionError( - {"error": type(error).__name__, "message": str(error)} - ) from error - return project_json(result) +if TYPE_CHECKING: + from inkcre_core.business.graph_navigation_retrieval.tools import * diff --git a/app/business/info_base/__init__.py b/app/business/info_base/__init__.py index 72c49b13..c1ea652b 100644 --- a/app/business/info_base/__init__.py +++ b/app/business/info_base/__init__.py @@ -1,14 +1,3 @@ -from .main import InfoBaseManager -from .commands import persist_graph, submit_graph, persist_stars, submit_stars -from .services import BlockService, RelationService, get_entity_records +"""Legacy import; use inkcre_core for new consumers.""" -__all__ = [ - "InfoBaseManager", - "persist_graph", - "submit_graph", - "persist_stars", - "submit_stars", - "get_entity_records", - "BlockService", - "RelationService", -] +from inkcre_core.business.info_base import * diff --git a/app/business/info_base/commands.py b/app/business/info_base/commands.py index 926847d1..90d158d4 100644 --- a/app/business/info_base/commands.py +++ b/app/business/info_base/commands.py @@ -1,79 +1,10 @@ -"""Graph application operations shared by transports and composing use cases.""" +"""Legacy import; use inkcre_core for new consumers.""" -from app.schemas.info_base.main import ( - GraphBlockIDMapping, - GraphForm, - SubmitGraphResult, - StarsGraphForm, -) -from app.schemas.info_base.block import BlockModel -from app.schemas.info_base.relation import RelationModel -from app.schemas.info_base.relation import RelationCreateForm +from typing import TYPE_CHECKING +from importlib import import_module +import sys -from app.persistence.info_base.uow import GraphUnitOfWork, graph_uow +sys.modules[__name__] = import_module("inkcre_core.business.info_base.commands") - -async def persist_graph(graph: GraphForm, uow: GraphUnitOfWork) -> SubmitGraphResult: - """Add a flat graph to the supplied unit of work without ending its transaction.""" - blocks = await uow.blocks.create_many(graph.blocks) - mappings: list[GraphBlockIDMapping] = [] - for form, block in zip(graph.blocks, blocks, strict=True): - if block.id is None: - raise RuntimeError("Inserted Block is missing its database ID") - mappings.append(GraphBlockIDMapping(local_id=form.id, id=block.id)) - persisted_ids = {mapping.local_id: mapping.id for mapping in mappings} - await uow.relations.create_many( - RelationCreateForm( - from_=persisted_ids.get(form.from_, form.from_), - to_=persisted_ids.get(form.to_, form.to_), - content=form.content, - ) - for form in graph.relations - ) - return SubmitGraphResult(blocks=tuple(mappings)) - - -async def submit_graph(graph: GraphForm) -> SubmitGraphResult: - """Commit one complete flat graph before returning its Block identity mapping.""" - async with graph_uow() as uow: - result = await persist_graph(graph, uow) - return result - - -async def get_related_block( - block_id: int, *, content: str, outgoing: bool = True -) -> BlockModel | None: - async with graph_uow() as uow: - return await uow.blocks.get_related(block_id, content=content, outgoing=outgoing) - - -async def persist_stars(graph: StarsGraphForm, uow: GraphUnitOfWork) -> BlockModel: - """Reconcile exact resolver identities and relations within one transaction.""" - from .resolver import ResolverManager - - candidate = BlockModel.model_validate(graph.block) - existing = await ResolverManager.get(candidate).get_existing_async(uow.blocks) - block = existing if existing is not None else await uow.blocks.create(graph.block) - if block.id is None: - raise RuntimeError("Persisted star Block is missing its database ID") - for arc in graph.out_arcs: - target = await persist_stars(arc.to_graph, uow) - if target.id is None: - raise RuntimeError("Persisted star Block is missing its database ID") - await uow.relations.fetchsert( - RelationModel(from_=block.id, to_=target.id, content=arc.relation.content) - ) - for arc in graph.in_arcs: - source = await persist_stars(arc.from_graph, uow) - if source.id is None: - raise RuntimeError("Persisted star Block is missing its database ID") - await uow.relations.fetchsert( - RelationModel(from_=source.id, to_=block.id, content=arc.relation.content) - ) - return block - - -async def submit_stars(graph: StarsGraphForm) -> BlockModel: - async with graph_uow() as uow: - result = await persist_stars(graph, uow) - return result +if TYPE_CHECKING: + from inkcre_core.business.info_base.commands import * diff --git a/app/business/info_base/main.py b/app/business/info_base/main.py index 8f1fd428..ef1ff3b1 100644 --- a/app/business/info_base/main.py +++ b/app/business/info_base/main.py @@ -1,117 +1,10 @@ -"""Info-base graph command coordination and cross-retrieval use cases.""" +"""Legacy import; use inkcre_core for new consumers.""" -import asyncio -from dataclasses import dataclass -import typing +from typing import TYPE_CHECKING +from importlib import import_module +import sys -from app.business.info_base.services import BlockService, get_entity_records -from app.business.lexical_retrieval import LexicalRetrievalManager -from app.business.semantic_retrieval import SemanticRetrievalManager -from app.schemas.lexical_retrieval import LexicalRetrievalResult -from app.schemas.semantic_retrieval import SemanticRetrievalResult, VectorRetrievalOptions -from app.schemas.info_base.block import BlockModel -from app.schemas.info_base.relation import RelationModel +sys.modules[__name__] = import_module("inkcre_core.business.info_base.main") -from app.schemas.info_base.main import ( - GraphBlockForm, - GraphForm, - GraphRelationForm, - StarsGraphForm, -) - - -@dataclass(frozen=True) -class RetrievalBranch: - result: LexicalRetrievalResult | SemanticRetrievalResult | None = None - error: BaseException | None = None - - -class InfoBaseManager: - """Own graph commands and use cases that span info-base read capabilities.""" - - @classmethod - async def retrieve( - cls, - query: str, - mode: typing.Literal["lexical", "semantic", "hybrid"] = "hybrid", - limit: int = 20, - ) -> dict[str, RetrievalBranch]: - """Run selected retrieval owners independently without merging their ranking.""" - operations: dict[str, typing.Awaitable] = {} - if mode in {"lexical", "hybrid"}: - operations["lexical"] = LexicalRetrievalManager.retrieve_local(query, limit) - if mode in {"semantic", "hybrid"}: - operations["semantic"] = SemanticRetrievalManager.retrieve_local( - query, options=VectorRetrievalOptions(limit=limit) - ) - outcomes = await asyncio.gather(*operations.values(), return_exceptions=True) - return { - name: RetrievalBranch( - error=outcome if isinstance(outcome, BaseException) else None, - result=None if isinstance(outcome, BaseException) else outcome, - ) - for name, outcome in zip(operations, outcomes, strict=True) - } - - @classmethod - async def get_entities( - cls, - entities: typing.Collection[tuple[typing.Literal["block", "relation"], int]], - *, - random_count: int = 1, - ) -> tuple[BlockModel | RelationModel | None, ...]: - """Read ordered persisted entities, or random Blocks when selection is empty.""" - if not entities: - return typing.cast( - tuple[BlockModel | RelationModel | None, ...], - await BlockService.get_random_many(random_count), - ) - blocks, relations = await get_entity_records( - tuple(identity for kind, identity in entities if kind == "block"), - tuple(identity for kind, identity in entities if kind == "relation"), - ) - return tuple( - blocks.get(identity) if kind == "block" else relations.get(identity) - for kind, identity in entities - ) - - @classmethod - def normalize_graph( - cls, - stars: StarsGraphForm, - id_start: int = -1, - ) -> GraphForm: - """Flatten recursive authoring while allocating deterministic negative IDs.""" - if id_start >= 0: - raise ValueError("id_start must be negative") - - next_id = id_start - blocks: list[GraphBlockForm] = [] - relations: list[GraphRelationForm] = [] - - def visit(star: StarsGraphForm) -> int: - nonlocal next_id - block_id = next_id - next_id -= 1 - blocks.append(GraphBlockForm.model_validate(star.block, update={"id": block_id})) - - for arc in star.out_arcs: - to_id = visit(arc.to_graph) - relations.append( - GraphRelationForm.model_validate( - arc.relation, - update={"from_": block_id, "to_": to_id}, - ) - ) - for arc in star.in_arcs: - from_id = visit(arc.from_graph) - relations.append( - GraphRelationForm.model_validate( - arc.relation, - update={"from_": from_id, "to_": block_id}, - ) - ) - return block_id - - visit(stars) - return GraphForm(blocks=tuple(blocks), relations=tuple(relations)) +if TYPE_CHECKING: + from inkcre_core.business.info_base.main import * diff --git a/app/business/info_base/resolver/__init__.py b/app/business/info_base/resolver/__init__.py index edf45ec6..fcc64c12 100644 --- a/app/business/info_base/resolver/__init__.py +++ b/app/business/info_base/resolver/__init__.py @@ -1,60 +1,3 @@ -from .bootstrap import register_core_resolvers -from .contracts import ( - CORE_RESOLVER_IDS, - CoreResolverID, - DuplicateResolverRegistrationError, - ResolverContentError, - ResolverContractError, - TextProjectionContext, - UnknownDraftResolverError, - UnknownResolverError, - UnsupportedResolverCapability, - ResolverMethodInputError, - UnknownResolverMethodError, -) -from .main import ( - Resolver, - ResolverDraftCapability, - ResolverManager, - ResolverMethodContract, -) +"""Legacy import; use inkcre_core for new consumers.""" -__all__ = [ - "ResolverManager", - "Resolver", - "register_core_resolvers", - "CORE_RESOLVER_IDS", - "CoreResolverID", - "ResolverContractError", - "TextProjectionContext", - "UnknownResolverError", - "UnknownDraftResolverError", - "DuplicateResolverRegistrationError", - "ResolverContentError", - "UnsupportedResolverCapability", - "ResolverDraftCapability", - "ResolverMethodContract", - "ResolverMethodInputError", - "UnknownResolverMethodError", - "AudioResolver", - "EPUBResolver", - "FileResolver", - "ImageResolver", - "PDFResolver", - "VideoResolver", - "TextResolver", - "MarkdownResolver", - "HTMLResolver", - "ZIPResolver", -] - -from .audio import AudioResolver -from .epub import EPUBResolver -from .file import FileResolver -from .image import ImageResolver -from .pdf import PDFResolver -from .video import VideoResolver -from .text import TextResolver -from .markdown import MarkdownResolver -from .html import HTMLResolver -from .zip import ZIPResolver +from inkcre_core.business.info_base.resolver import * diff --git a/app/business/info_base/resolver/audio.py b/app/business/info_base/resolver/audio.py index e5ce2fa4..ee83732b 100644 --- a/app/business/info_base/resolver/audio.py +++ b/app/business/info_base/resolver/audio.py @@ -1,170 +1,10 @@ -"""Exact audio resolver using metadata-only PyAV inspection.""" +"""Legacy import; use inkcre_core for new consumers.""" -import asyncio -from dataclasses import dataclass -from io import BytesIO -import typing +from typing import TYPE_CHECKING +from importlib import import_module +import sys -import av -from av.stream import Disposition -import pydantic +sys.modules[__name__] = import_module("inkcre_core.business.info_base.resolver.audio") -from app.business.deployment_config import DeploymentConfigManager, DeploymentConfigService -from app.schemas.ai import AudioContentPart - -from .contracts import ( - ResolverContentError, - TextProjectionContext, - UnsupportedResolverCapability, -) -from .inspection import ( - ByteContentFacts, - detect_media_type, - format_lexical_facts, - require_bytes, -) -from .main import Resolver -from .materialization import try_materialize_model_text - - -AUDIO_RESOLVER_CONFIG_KEY = "core.resolver.audio" -AUDIO_RESOLVER_CONFIG_SCHEMA = "core.resolver.audio.config.v1" - - -class AudioResolverConfig(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - transcript_model: int - - -DeploymentConfigManager.register_schema( - AUDIO_RESOLVER_CONFIG_SCHEMA, AudioResolverConfig, keys=(AUDIO_RESOLVER_CONFIG_KEY,) -) - - -@dataclass(frozen=True, slots=True) -class AudioSolvedContent(ByteContentFacts): - container: str | None - codec: str | None - duration_ms: int | None - channels: int | None - sample_rate_hz: int | None - bitrate_bps: int | None - - -def _container_name(format_name: str | None) -> str | None: - if not format_name: - return None - names = format_name.split(",") - return "mp4" if "mp4" in names else names[0] - - -def _duration_ms(container, stream) -> int | None: - if stream.duration is not None and stream.time_base is not None: - return round(float(stream.duration * stream.time_base) * 1000) - if container.duration is not None: - return round(container.duration / 1000) - return None - - -def _primary_stream(streams): - default = [stream for stream in streams if bool(stream.disposition & Disposition.default)] - ordinary = [ - stream for stream in streams if not bool(stream.disposition & Disposition.attached_pic) - ] - candidates = default or ordinary or list(streams) - return candidates[0] if candidates else None - - -def _inspect_audio(content: bytes) -> AudioSolvedContent: - try: - with av.open(BytesIO(content), mode="r") as container: - stream = _primary_stream(container.streams.audio) - if stream is None: - raise ResolverContentError("core.audio.v1", "no audio stream") - codec = stream.codec_context - facts = AudioSolvedContent( - content=content, - byte_size=len(content), - detected_media_type=detect_media_type(content), - container=_container_name(container.format.name), - codec=codec.name, - duration_ms=_duration_ms(container, stream), - channels=codec.channels, - sample_rate_hz=codec.sample_rate, - bitrate_bps=stream.bit_rate or container.bit_rate, - ) - except ResolverContentError: - raise - except (av.FFmpegError, OSError, ValueError) as error: - raise ResolverContentError("core.audio.v1", "invalid audio") from error - return facts - - -class AudioResolver( - Resolver[AudioSolvedContent, bytes], - rso_type="core.audio.v1", -): - async def _get_solved_content( - self, - *, - refresh: bool = False, - materialize_missing: bool = True, - ) -> AudioSolvedContent: - del materialize_missing - content = require_bytes( - await self.get_raw_content(refresh=refresh), - self.__rsotype__, - ) - return await asyncio.to_thread(_inspect_audio, content) - - async def get_text( - self, - *, - context: TextProjectionContext = "default", - refresh: bool = False, - materialize_missing: bool = True, - ) -> str: - if context == "default": - raise UnsupportedResolverCapability(self.__rsotype__, "text") - solved = await self.get_solved_content(refresh=refresh, materialize_missing=False) - if materialize_missing: - config = typing.cast( - AudioResolverConfig | None, - await DeploymentConfigService.get(AUDIO_RESOLVER_CONFIG_KEY), - ) - if config is not None: - media_type = solved.detected_media_type or ( - f"audio/{solved.container}" if solved.container is not None else None - ) - if media_type is not None: - await try_materialize_model_text( - block_id=self.block_id, - role="transcript", - model=config.transcript_model, - instruction=( - "Transcribe only speech and authored spoken language in the audio. " - "Return plain text with no summary or explanation." - ), - media=AudioContentPart( - data=solved.content, - mime_type=media_type, - transfer_url=await self.get_transfer_url(), - ), - ) - return format_lexical_facts( - "audio", - ( - ("media type", solved.detected_media_type), - ("container", solved.container), - ("codec", solved.codec), - ("duration ms", solved.duration_ms), - ("channels", solved.channels), - ("sample rate Hz", solved.sample_rate_hz), - ("bitrate bps", solved.bitrate_bps), - ), - ) - - async def get_label(self, *, refresh: bool = False) -> str: - del refresh - return "audio" +if TYPE_CHECKING: + from inkcre_core.business.info_base.resolver.audio import * diff --git a/app/business/info_base/resolver/bootstrap.py b/app/business/info_base/resolver/bootstrap.py index 862a2379..cbae51ed 100644 --- a/app/business/info_base/resolver/bootstrap.py +++ b/app/business/info_base/resolver/bootstrap.py @@ -1,31 +1,10 @@ -"""Explicit registration of resolver decoders owned by impl-py.""" +"""Legacy import; use inkcre_core for new consumers.""" -from .main import ResolverManager +from typing import TYPE_CHECKING +from importlib import import_module +import sys +sys.modules[__name__] = import_module("inkcre_core.business.info_base.resolver.bootstrap") -def register_core_resolvers() -> None: - """Register built-in decoders independently of extension loading.""" - from .audio import AudioResolver - from .epub import EPUBResolver - from .file import FileResolver - from .html import HTMLResolver - from .image import ImageResolver - from .pdf import PDFResolver - from .text import TextResolver - from .markdown import MarkdownResolver - from .video import VideoResolver - from .zip import ZIPResolver - - for resolver_class in ( - TextResolver, - MarkdownResolver, - HTMLResolver, - ImageResolver, - AudioResolver, - VideoResolver, - PDFResolver, - EPUBResolver, - ZIPResolver, - FileResolver, - ): - ResolverManager.register_resolver(resolver_class) +if TYPE_CHECKING: + from inkcre_core.business.info_base.resolver.bootstrap import * diff --git a/app/business/info_base/resolver/contracts.py b/app/business/info_base/resolver/contracts.py index fbcfa89b..63abb2f6 100644 --- a/app/business/info_base/resolver/contracts.py +++ b/app/business/info_base/resolver/contracts.py @@ -1,95 +1,10 @@ -"""Stable resolver IDs, projection controls, and typed contract failures.""" +"""Legacy import; use inkcre_core for new consumers.""" -from typing import Literal, TypeAlias +from typing import TYPE_CHECKING +from importlib import import_module +import sys -import pydantic +sys.modules[__name__] = import_module("inkcre_core.business.info_base.resolver.contracts") - -TextProjectionContext: TypeAlias = Literal["default", "lexical"] - - -CoreResolverID: TypeAlias = Literal[ - "core.text.v1", - "core.markdown.v1", - "core.html.v1", - "core.image.v1", - "core.audio.v1", - "core.video.v1", - "core.pdf.v1", - "core.epub.v1", - "core.zip.v1", - "core.file.v1", -] - -CORE_RESOLVER_IDS: tuple[CoreResolverID, ...] = ( - "core.text.v1", - "core.markdown.v1", - "core.html.v1", - "core.image.v1", - "core.audio.v1", - "core.video.v1", - "core.pdf.v1", - "core.epub.v1", - "core.zip.v1", - "core.file.v1", -) - - -class ResolverContractError(RuntimeError): - """Base failure for resolver selection and capability contracts.""" - - -class ResolverMethodInputError(pydantic.ValidationError): - """Native input details, distinguishable from errors inside a content method.""" - - -class UnknownResolverMethodError(ResolverContractError, ValueError): - """An exact Resolver has no projected read method with this name.""" - - -class UnknownResolverError(ResolverContractError): - """No exact resolver decoder is registered for a persisted resolver ID.""" - - def __init__(self, resolver_id: str): - self.resolver_id = resolver_id - super().__init__(f"Unknown resolver ID: {resolver_id}") - - -class UnknownDraftResolverError(ResolverContractError): - """The selected exact Resolver has no graph-drafting capability.""" - - def __init__(self, resolver_id: str): - self.resolver_id = resolver_id - super().__init__(f"Resolver is not available for graph drafting: {resolver_id}") - - -class DuplicateResolverRegistrationError(ResolverContractError): - """Two different decoder classes claimed the same exact resolver ID.""" - - def __init__(self, resolver_id: str, existing: type, attempted: type): - self.resolver_id = resolver_id - self.existing = existing - self.attempted = attempted - super().__init__( - f"Resolver ID {resolver_id} is already registered by " - f"{existing.__module__}.{existing.__qualname__}; " - f"cannot register {attempted.__module__}.{attempted.__qualname__}" - ) - - -class UnsupportedResolverCapability(ResolverContractError): - """The exact resolver contract does not provide one requested projection.""" - - def __init__(self, resolver_id: str, capability: str): - self.resolver_id = resolver_id - self.capability = capability - super().__init__(f"Resolver {resolver_id} does not support {capability}") - - -class ResolverContentError(ResolverContractError, ValueError): - """Hydrated content does not satisfy its exact resolver contract.""" - - def __init__(self, resolver_id: str, reason: str): - self.resolver_id = resolver_id - self.reason = reason - super().__init__(f"Resolver {resolver_id} rejected content: {reason}") +if TYPE_CHECKING: + from inkcre_core.business.info_base.resolver.contracts import * diff --git a/app/business/info_base/resolver/epub.py b/app/business/info_base/resolver/epub.py index a403579e..793e672a 100644 --- a/app/business/info_base/resolver/epub.py +++ b/app/business/info_base/resolver/epub.py @@ -1,219 +1,10 @@ -"""Exact EPUB resolver using bounded package-document inspection.""" +"""Legacy import; use inkcre_core for new consumers.""" -import asyncio -import datetime -from dataclasses import dataclass -from io import BytesIO -import zipfile +from typing import TYPE_CHECKING +from importlib import import_module +import sys -from lxml import etree +sys.modules[__name__] = import_module("inkcre_core.business.info_base.resolver.epub") -from .contracts import ( - ResolverContentError, - TextProjectionContext, - UnsupportedResolverCapability, -) -from .inspection import ByteContentFacts, format_lexical_facts, require_bytes -from .label import format_label -from .main import Resolver - - -_MAX_EPUB_MEMBERS = 10_000 -_MAX_PACKAGE_BYTES = 2 * 1024 * 1024 -_XML_PARSER = etree.XMLParser( - resolve_entities=False, - no_network=True, - recover=False, - huge_tree=False, -) - - -@dataclass(frozen=True, slots=True) -class EPUBSolvedContent(ByteContentFacts): - epub_version: str | None - title: str | None - creators: tuple[str, ...] - languages: tuple[str, ...] - modified_at: datetime.datetime | None - manifest_count: int - spine_count: int - has_navigation: bool - - -def _local_name(element: etree._Element) -> str: - return etree.QName(element).localname - - -def _read_bounded( - archive: zipfile.ZipFile, - member_name: str, - *, - limit: int = _MAX_PACKAGE_BYTES, -) -> bytes: - try: - member = archive.getinfo(member_name) - except KeyError as error: - raise ResolverContentError( - "core.epub.v1", - f"missing required EPUB member {member_name}", - ) from error - if member.file_size > limit: - raise ResolverContentError( - "core.epub.v1", - f"EPUB member {member_name} exceeds inspection limit", - ) - with archive.open(member) as stream: - result = stream.read(limit + 1) - if len(result) > limit: - raise ResolverContentError( - "core.epub.v1", - f"EPUB member {member_name} exceeds inspection limit", - ) - return result - - -def _parse_xml(content: bytes, label: str) -> etree._Element: - try: - return etree.fromstring(content, parser=_XML_PARSER) - except etree.XMLSyntaxError as error: - raise ResolverContentError("core.epub.v1", f"invalid {label}") from error - - -def _first_text(elements: list[etree._Element]) -> str | None: - for element in elements: - if element.text and (value := element.text.strip()): - return value - return None - - -def _all_text(elements: list[etree._Element]) -> tuple[str, ...]: - return tuple( - value for element in elements if element.text and (value := element.text.strip()) - ) - - -def _parse_modified(value: str | None) -> datetime.datetime | None: - if value is None: - return None - try: - parsed = datetime.datetime.fromisoformat(value.replace("Z", "+00:00")) - except ValueError as error: - raise ResolverContentError( - "core.epub.v1", - "invalid dcterms:modified timestamp", - ) from error - return parsed - - -def _inspect_epub(content: bytes) -> EPUBSolvedContent: - try: - with zipfile.ZipFile(BytesIO(content)) as archive: - if len(archive.infolist()) > _MAX_EPUB_MEMBERS: - raise ResolverContentError("core.epub.v1", "too many EPUB members") - if _read_bounded(archive, "mimetype", limit=128) != b"application/epub+zip": - raise ResolverContentError("core.epub.v1", "invalid EPUB mimetype member") - - container = _parse_xml( - _read_bounded(archive, "META-INF/container.xml"), - "EPUB container document", - ) - rootfiles = [ - element for element in container.iter() if _local_name(element) == "rootfile" - ] - package_path = next( - (element.get("full-path") for element in rootfiles if element.get("full-path")), - None, - ) - if package_path is None: - raise ResolverContentError("core.epub.v1", "missing EPUB package path") - package = _parse_xml( - _read_bounded(archive, package_path), - "EPUB package document", - ) - except ResolverContentError: - raise - except (zipfile.BadZipFile, OSError, ValueError) as error: - raise ResolverContentError("core.epub.v1", "invalid EPUB container") from error - - elements = list(package.iter()) - titles = [element for element in elements if _local_name(element) == "title"] - creators = [element for element in elements if _local_name(element) == "creator"] - languages = [element for element in elements if _local_name(element) == "language"] - manifest_items = [element for element in elements if _local_name(element) == "item"] - spine_items = [element for element in elements if _local_name(element) == "itemref"] - modified = next( - ( - element.text.strip() - for element in elements - if _local_name(element) == "meta" - and element.get("property") == "dcterms:modified" - and element.text - and element.text.strip() - ), - None, - ) - - return EPUBSolvedContent( - content=content, - byte_size=len(content), - detected_media_type="application/epub+zip", - epub_version=package.get("version"), - title=_first_text(titles), - creators=_all_text(creators), - languages=_all_text(languages), - modified_at=_parse_modified(modified), - manifest_count=len(manifest_items), - spine_count=len(spine_items), - has_navigation=any( - "nav" in (element.get("properties") or "").split() for element in manifest_items - ), - ) - - -class EPUBResolver( - Resolver[EPUBSolvedContent, bytes], - rso_type="core.epub.v1", -): - async def _get_solved_content( - self, - *, - refresh: bool = False, - materialize_missing: bool = True, - ) -> EPUBSolvedContent: - del materialize_missing - content = require_bytes( - await self.get_raw_content(refresh=refresh), - self.__rsotype__, - ) - return await asyncio.to_thread(_inspect_epub, content) - - async def get_text( - self, - *, - context: TextProjectionContext = "default", - refresh: bool = False, - materialize_missing: bool = True, - ) -> str: - if context == "default": - raise UnsupportedResolverCapability(self.__rsotype__, "text") - del materialize_missing - solved = await self.get_solved_content(refresh=refresh, materialize_missing=False) - return format_lexical_facts( - "EPUB", - ( - ("media type", solved.detected_media_type), - ("title", solved.title), - ("creators", ", ".join(solved.creators) or None), - ("languages", ", ".join(solved.languages) or None), - ("EPUB version", solved.epub_version), - ("manifest items", solved.manifest_count), - ("spine items", solved.spine_count), - ), - ) - - async def get_label(self, *, refresh: bool = False) -> str: - solved = await self.get_solved_content( - refresh=refresh, - materialize_missing=False, - ) - return format_label("EPUB", solved.title) +if TYPE_CHECKING: + from inkcre_core.business.info_base.resolver.epub import * diff --git a/app/business/info_base/resolver/file.py b/app/business/info_base/resolver/file.py index b95167f7..84459095 100644 --- a/app/business/info_base/resolver/file.py +++ b/app/business/info_base/resolver/file.py @@ -1,64 +1,10 @@ -"""Exact generic-file resolver for bytes without stronger semantics.""" +"""Legacy import; use inkcre_core for new consumers.""" -import asyncio -from dataclasses import dataclass +from typing import TYPE_CHECKING +from importlib import import_module +import sys -from .contracts import TextProjectionContext, UnsupportedResolverCapability -from .inspection import ( - ByteContentFacts, - detect_media_type, - format_lexical_facts, - require_bytes, -) -from .main import Resolver +sys.modules[__name__] = import_module("inkcre_core.business.info_base.resolver.file") - -@dataclass(frozen=True, slots=True) -class FileSolvedContent(ByteContentFacts): - pass - - -def _inspect_file(content: bytes) -> FileSolvedContent: - return FileSolvedContent( - content=content, - byte_size=len(content), - detected_media_type=detect_media_type(content), - ) - - -class FileResolver( - Resolver[FileSolvedContent, bytes], - rso_type="core.file.v1", -): - async def _get_solved_content( - self, - *, - refresh: bool = False, - materialize_missing: bool = True, - ) -> FileSolvedContent: - del materialize_missing - content = require_bytes( - await self.get_raw_content(refresh=refresh), - self.__rsotype__, - ) - return await asyncio.to_thread(_inspect_file, content) - - async def get_text( - self, - *, - context: TextProjectionContext = "default", - refresh: bool = False, - materialize_missing: bool = True, - ) -> str: - if context == "default": - raise UnsupportedResolverCapability(self.__rsotype__, "text") - del materialize_missing - solved = await self.get_solved_content(refresh=refresh, materialize_missing=False) - return format_lexical_facts( - "file", - (("media type", solved.detected_media_type), ("bytes", solved.byte_size)), - ) - - async def get_label(self, *, refresh: bool = False) -> str: - del refresh - return "file" +if TYPE_CHECKING: + from inkcre_core.business.info_base.resolver.file import * diff --git a/app/business/info_base/resolver/html.py b/app/business/info_base/resolver/html.py index ad506485..3dece393 100644 --- a/app/business/info_base/resolver/html.py +++ b/app/business/info_base/resolver/html.py @@ -1,73 +1,10 @@ -"""Exact HTML-source resolver with a derived text projection.""" +"""Legacy import; use inkcre_core for new consumers.""" -import asyncio +from typing import TYPE_CHECKING +from importlib import import_module +import sys -import html2text -from lxml import html as lxml_html +sys.modules[__name__] = import_module("inkcre_core.business.info_base.resolver.html") -from .inspection import decode_html_bytes -from .label import format_label -from .main import Resolver, TextProjectionContext - - -class HTMLResolver(Resolver[str, str | bytes], rso_type="core.html.v1"): - @classmethod - def create_graph(cls, url: str): - from app.schemas.info_base.block import BlockForm - from app.schemas.info_base.main import StarsGraphForm - - return StarsGraphForm( - block=BlockForm(resolver=cls.__rsotype__, content=url, storage=-1) - ) - - async def _get_solved_content( - self, - *, - refresh: bool = False, - materialize_missing: bool = True, - ) -> str: - del materialize_missing - content = await self.get_raw_content(refresh=refresh) - return ( - content if isinstance(content, str) else decode_html_bytes(content, self.__rsotype__) - ) - - async def get_text( - self, - *, - context: TextProjectionContext = "default", - refresh: bool = False, - materialize_missing: bool = True, - ) -> str: - del context - source = await self.get_solved_content( - refresh=refresh, - materialize_missing=materialize_missing, - ) - if any(f"<{tag}" in source.lower() for tag in ("video", "audio", "track")): - document = lxml_html.document_fromstring(source) - for resource in document.xpath("//video | //audio | //source | //track"): - for attribute in ("src", "poster"): - if address := resource.get(attribute): - link = lxml_html.Element("a", href=address) - link.text = resource.get("label") or address - resource.addnext(link) - source = lxml_html.tostring(document, encoding="unicode") - return await asyncio.to_thread(html2text.HTML2Text().handle, source) - - async def get_label(self, *, refresh: bool = False) -> str: - source = await self.get_solved_content( - refresh=refresh, - materialize_missing=False, - ) - identifier = None - try: - document = lxml_html.fromstring(source) - values = document.xpath("//title/text()") or document.xpath( - "//h1//text() | //h2//text() | //h3//text() | //h4//text() | " - "//h5//text() | //h6//text()" - ) - identifier = next((str(value) for value in values if str(value).strip()), None) - except (TypeError, ValueError): - pass - return format_label("html", identifier) +if TYPE_CHECKING: + from inkcre_core.business.info_base.resolver.html import * diff --git a/app/business/info_base/resolver/image.py b/app/business/info_base/resolver/image.py index 3f4e5765..53727271 100644 --- a/app/business/info_base/resolver/image.py +++ b/app/business/info_base/resolver/image.py @@ -1,155 +1,10 @@ -"""Exact image resolver using bounded Pillow header inspection.""" +"""Legacy import; use inkcre_core for new consumers.""" -import asyncio -from dataclasses import dataclass -from io import BytesIO -import typing +from typing import TYPE_CHECKING +from importlib import import_module +import sys -from PIL import Image, UnidentifiedImageError -import pydantic +sys.modules[__name__] = import_module("inkcre_core.business.info_base.resolver.image") -from app.business.deployment_config import DeploymentConfigManager, DeploymentConfigService -from app.schemas.ai import ImageContentPart -from app.schemas.info_base.block import BlockForm -from app.schemas.info_base.main import OutArcForm, StarsGraphForm -from app.schemas.info_base.relation import RelationForm - -from .contracts import ( - ResolverContentError, - TextProjectionContext, - UnsupportedResolverCapability, -) -from .inspection import ( - ByteContentFacts, - detect_media_type, - format_lexical_facts, - require_bytes, -) -from .main import Resolver -from .materialization import try_materialize_model_text - - -IMAGE_RESOLVER_CONFIG_KEY = "core.resolver.image" -IMAGE_RESOLVER_CONFIG_SCHEMA = "core.resolver.image.config.v1" - - -class ImageResolverConfig(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - text_model: int - - -DeploymentConfigManager.register_schema( - IMAGE_RESOLVER_CONFIG_SCHEMA, ImageResolverConfig, keys=(IMAGE_RESOLVER_CONFIG_KEY,) -) - - -@dataclass(frozen=True, slots=True) -class ImageSolvedContent(ByteContentFacts): - format: str | None - width: int | None - height: int | None - frame_count: int | None - - -def _inspect_image(content: bytes) -> ImageSolvedContent: - try: - with Image.open(BytesIO(content)) as image: - image_format = image.format.lower() if image.format else None - width, height = image.size - frame_count = getattr(image, "n_frames", None) - image.verify() - except (UnidentifiedImageError, OSError, SyntaxError, ValueError) as error: - raise ResolverContentError("core.image.v1", "invalid image") from error - return ImageSolvedContent( - content=content, - byte_size=len(content), - detected_media_type=detect_media_type(content), - format=image_format, - width=width, - height=height, - frame_count=frame_count, - ) - - -class ImageResolver( - Resolver[ImageSolvedContent, bytes], - rso_type="core.image.v1", -): - @classmethod - def create_graph(cls, url: str, alt_text: str | None = None) -> StarsGraphForm: - from .text import TextResolver - - return StarsGraphForm( - block=BlockForm(resolver=cls.__rsotype__, content=url, storage=-1), - out_arcs=( - OutArcForm( - relation=RelationForm(content="alt:text"), - to_graph=TextResolver.create_graph(alt_text), - ), - ) - if alt_text - else (), - ) - - async def _get_solved_content( - self, - *, - refresh: bool = False, - materialize_missing: bool = True, - ) -> ImageSolvedContent: - del materialize_missing - content = require_bytes( - await self.get_raw_content(refresh=refresh), - self.__rsotype__, - ) - return await asyncio.to_thread(_inspect_image, content) - - async def get_text( - self, - *, - context: TextProjectionContext = "default", - refresh: bool = False, - materialize_missing: bool = True, - ) -> str: - if context == "default": - raise UnsupportedResolverCapability(self.__rsotype__, "text") - solved = await self.get_solved_content(refresh=refresh, materialize_missing=False) - if materialize_missing: - config = typing.cast( - ImageResolverConfig | None, - await DeploymentConfigService.get(IMAGE_RESOLVER_CONFIG_KEY), - ) - if config is not None: - media_type = solved.detected_media_type or ( - f"image/{solved.format}" if solved.format is not None else None - ) - if media_type is not None: - await try_materialize_model_text( - block_id=self.block_id, - role="text", - model=config.text_model, - instruction=( - "Transcribe only written text visibly present in the image. " - "Return plain text with no explanation; return no content when none exists." - ), - media=ImageContentPart( - data=solved.content, - mime_type=media_type, - transfer_url=await self.get_transfer_url(), - ), - ) - return format_lexical_facts( - "image", - ( - ("media type", solved.detected_media_type), - ("format", solved.format), - ("width", solved.width), - ("height", solved.height), - ("frames", solved.frame_count), - ), - ) - - async def get_label(self, *, refresh: bool = False) -> str: - del refresh - return "image" +if TYPE_CHECKING: + from inkcre_core.business.info_base.resolver.image import * diff --git a/app/business/info_base/resolver/inspection.py b/app/business/info_base/resolver/inspection.py index 4ac77f4f..b690a7d2 100644 --- a/app/business/info_base/resolver/inspection.py +++ b/app/business/info_base/resolver/inspection.py @@ -1,104 +1,10 @@ -"""Bounded implementation helpers shared by exact core content resolvers.""" +"""Legacy import; use inkcre_core for new consumers.""" -from dataclasses import dataclass -from collections.abc import Iterable -import codecs -import re +from typing import TYPE_CHECKING +from importlib import import_module +import sys -import puremagic +sys.modules[__name__] = import_module("inkcre_core.business.info_base.resolver.inspection") -from .contracts import ResolverContentError - - -@dataclass(frozen=True, slots=True) -class ByteContentFacts: - """Peer-local actual bytes plus common byte-derived facts.""" - - content: bytes - byte_size: int - detected_media_type: str | None - - -def detect_media_type(content: bytes) -> str | None: - """Return bounded signature detection without inventing an unknown value.""" - try: - detected = puremagic.from_string(content, mime=True) - except puremagic.PureError: - return None - return detected if isinstance(detected, str) else None - - -def decode_unicode_bytes(content: bytes, resolver_id: str) -> str: - """Decode a Unicode BOM when present, otherwise require strict UTF-8.""" - decoders = ( - (codecs.BOM_UTF32_LE, "utf-32"), - (codecs.BOM_UTF32_BE, "utf-32"), - (codecs.BOM_UTF8, "utf-8-sig"), - (codecs.BOM_UTF16_LE, "utf-16"), - (codecs.BOM_UTF16_BE, "utf-16"), - ) - encoding = next( - (candidate for bom, candidate in decoders if content.startswith(bom)), - "utf-8", - ) - try: - return content.decode(encoding, errors="strict") - except UnicodeError as error: - raise ResolverContentError(resolver_id, f"invalid {encoding} text") from error - - -_HTML_CHARSET_SEARCH_BYTES = 4096 -_HTML_CHARSET_PATTERNS = ( - re.compile(rb"]*charset\s*=\s*['\"]?\s*([a-zA-Z0-9._-]+)", re.I), - re.compile(rb"]*content\s*=\s*['\"][^'\"]*charset=([a-zA-Z0-9._-]+)", re.I), -) - - -def decode_html_bytes(content: bytes, resolver_id: str) -> str: - """Decode HTML using a Unicode BOM, bounded meta declaration, or strict UTF-8.""" - for bom in ( - codecs.BOM_UTF32_LE, - codecs.BOM_UTF32_BE, - codecs.BOM_UTF8, - codecs.BOM_UTF16_LE, - codecs.BOM_UTF16_BE, - ): - if content.startswith(bom): - return decode_unicode_bytes(content, resolver_id) - - prefix = content[:_HTML_CHARSET_SEARCH_BYTES] - declared_encoding = next( - ( - match.group(1).decode("ascii") - for pattern in _HTML_CHARSET_PATTERNS - if (match := pattern.search(prefix)) is not None - ), - "utf-8", - ) - try: - encoding = codecs.lookup(declared_encoding).name - except LookupError as error: - raise ResolverContentError( - resolver_id, - f"unknown declared HTML charset {declared_encoding}", - ) from error - try: - return content.decode(encoding, errors="strict") - except UnicodeError as error: - raise ResolverContentError(resolver_id, f"invalid {encoding} HTML") from error - - -def require_bytes(content: object, resolver_id: str) -> bytes: - if not isinstance(content, bytes): - raise ResolverContentError(resolver_id, "byte-oriented content must be storage-backed") - return content - - -def format_lexical_facts( - kind: str, - facts: Iterable[tuple[str, object | None]], -) -> str: - """Format bounded Resolver-owned metadata without serializing arbitrary objects.""" - lines = [kind] - lines.extend(f"{label}: {value}" for label, value in facts if value is not None) - return "\n".join(lines) +if TYPE_CHECKING: + from inkcre_core.business.info_base.resolver.inspection import * diff --git a/app/business/info_base/resolver/label.py b/app/business/info_base/resolver/label.py index a271cb2d..0da9422f 100644 --- a/app/business/info_base/resolver/label.py +++ b/app/business/info_base/resolver/label.py @@ -1,31 +1,10 @@ -"""Shared mechanics for concise Resolver-owned Block labels.""" +"""Legacy import; use inkcre_core for new consumers.""" -LABEL_IDENTIFIER_LIMIT = 96 +from typing import TYPE_CHECKING +from importlib import import_module +import sys +sys.modules[__name__] = import_module("inkcre_core.business.info_base.resolver.label") -def normalize_label_identifier( - value: str | None, - *, - first_line: bool = False, -) -> str | None: - """Normalize whitespace and bound one optional human-readable identifier.""" - if value is None: - return None - candidates = value.splitlines() if first_line else (value,) - normalized = next((" ".join(item.split()) for item in candidates if item.strip()), "") - if not normalized: - return None - if len(normalized) > LABEL_IDENTIFIER_LIMIT: - return f"{normalized[:LABEL_IDENTIFIER_LIMIT]}…" - return normalized - - -def format_label( - kind: str, - identifier: str | None = None, - *, - first_line: bool = False, -) -> str: - """Format one stable resolver-qualified label without exposing exact IDs.""" - normalized = normalize_label_identifier(identifier, first_line=first_line) - return kind if normalized is None else f"{kind} <{normalized}>" +if TYPE_CHECKING: + from inkcre_core.business.info_base.resolver.label import * diff --git a/app/business/info_base/resolver/main.py b/app/business/info_base/resolver/main.py index 57820783..0bb432e7 100644 --- a/app/business/info_base/resolver/main.py +++ b/app/business/info_base/resolver/main.py @@ -1,434 +1,10 @@ -import abc -from collections.abc import Collection -from dataclasses import dataclass -import inspect -import typing -from typing import Optional as Opt +"""Legacy import; use inkcre_core for new consumers.""" -import pydantic +from typing import TYPE_CHECKING +from importlib import import_module +import sys -from app.business.info_base.services import RelationService -from app.schemas.info_base.main import StarsGraphForm -from app.schemas.info_base.block import BlockForm, BlockID, ResolverType, BlockModel -from app.schemas.info_base.relation import RelationModel -from app.schemas.info_base.storage import StorageID +sys.modules[__name__] = import_module("inkcre_core.business.info_base.resolver.main") -from app.persistence.info_base.repository import BlockRepository - -from .contracts import ( - DuplicateResolverRegistrationError, - TextProjectionContext, - UnknownDraftResolverError, - UnknownResolverError, - UnknownResolverMethodError, - ResolverMethodInputError, -) - - -@dataclass(frozen=True) -class ResolverDraftCapability: - """One code-owned Resolver graph-drafting contract.""" - - resolver: ResolverType - description: str - input_model: type[pydantic.BaseModel] - resolver_cls: type["Resolver"] - - -@dataclass(frozen=True) -class ResolverMethodContract: - """One typed public read method on a registered Resolver.""" - - name: str - description: str - input_model: type[pydantic.BaseModel] - - @property - def input_schema(self) -> dict[str, typing.Any]: - return self.input_model.model_json_schema() - - -class ResolverManager: - RESOLVER_CLS: dict[ResolverType, type["Resolver"]] = {} - """Global resolver registry. - - Map ResolverType to Resolver class - """ - - @classmethod - def register_resolver(cls, resolver_cls: type["Resolver"]) -> None: - existing = cls.RESOLVER_CLS.get(resolver_cls.__rsotype__) - if existing is resolver_cls: - return - if existing is not None: - raise DuplicateResolverRegistrationError( - resolver_cls.__rsotype__, - existing, - resolver_cls, - ) - cls.RESOLVER_CLS[resolver_cls.__rsotype__] = resolver_cls - - @classmethod - def get(cls, block: BlockModel) -> "Resolver": - """Create resolver instance from block.""" - try: - resolver_cls = cls.RESOLVER_CLS[block.resolver] - except KeyError as error: - raise UnknownResolverError(block.resolver) from error - return resolver_cls(block) - - @classmethod - def get_draft_capabilities(cls) -> tuple[ResolverDraftCapability, ...]: - """Snapshot currently registered Resolvers that can draft rooted graphs.""" - capabilities = ( - ResolverDraftCapability( - resolver=resolver_id, - description=resolver_cls.draft_description, - input_model=resolver_cls.draft_input_model, - resolver_cls=resolver_cls, - ) - for resolver_id, resolver_cls in cls.RESOLVER_CLS.items() - if resolver_cls.draft_description is not None - and resolver_cls.draft_input_model is not None - ) - return tuple(sorted(capabilities, key=lambda capability: capability.resolver)) - - @classmethod - def get_draft_capability( - cls, - resolver: ResolverType, - ) -> ResolverDraftCapability: - """Select one exact graph-drafting contract without fallback.""" - for capability in cls.get_draft_capabilities(): - if capability.resolver == resolver: - return capability - raise UnknownDraftResolverError(resolver) - - @classmethod - def get_method_contracts( - cls, - resolver: ResolverType, - ) -> tuple[ResolverMethodContract, ...]: - """Discover typed public read methods on one registered Resolver.""" - resolver_cls = cls.RESOLVER_CLS.get(resolver) - if resolver_cls is None: - return () - return cls._method_contracts(resolver_cls) - - @classmethod - def get_common_method_contracts(cls) -> tuple[ResolverMethodContract, ...]: - """Common Resolver reads, available without per-type discovery.""" - return cls._method_contracts(Resolver) - - @staticmethod - def _method_contracts( - resolver_cls: type["Resolver"], - ) -> tuple[ResolverMethodContract, ...]: - contracts: list[ResolverMethodContract] = [] - for name, function in inspect.getmembers(resolver_cls, predicate=inspect.isfunction): - if name.startswith("_") or not name.startswith(("get_", "read_")): - continue - try: - signature = inspect.signature(function, eval_str=True) - fields: dict[str, tuple[typing.Any, typing.Any]] = {} - for parameter in signature.parameters.values(): - if parameter.name == "self": - continue - if parameter.kind in (parameter.VAR_POSITIONAL, parameter.VAR_KEYWORD): - raise TypeError("Variadic Resolver methods are not projectable") - if parameter.annotation is inspect.Parameter.empty: - raise TypeError("Resolver method parameters must be typed") - default = ( - ... if parameter.default is inspect.Parameter.empty else parameter.default - ) - annotation = parameter.annotation - if typing.get_origin(annotation) is Collection: - item_type = typing.get_args(annotation)[0] - annotation = tuple[item_type, ...] - fields[parameter.name] = (annotation, default) - input_model = typing.cast(typing.Any, pydantic.create_model)( - f"{resolver_cls.__name__}_{name}_Arguments", - __config__=pydantic.ConfigDict(extra="forbid"), - **fields, - ) - input_model.model_json_schema() - except (NameError, TypeError, pydantic.PydanticSchemaGenerationError): - continue - contracts.append( - ResolverMethodContract( - name=name, - description=inspect.getdoc(function) or name.replace("_", " "), - input_model=input_model, - ) - ) - return tuple(contracts) - - @classmethod - def get_method_contract( - cls, - resolver: ResolverType, - name: str, - ) -> ResolverMethodContract | None: - return next( - ( - contract for contract in cls.get_method_contracts(resolver) if contract.name == name - ), - None, - ) - - @classmethod - async def invoke_method( - cls, - block: BlockModel, - name: str, - arguments: dict[str, typing.Any], - ) -> typing.Any: - """Validate and invoke one projected read method on an exact Block Resolver.""" - resolver = cls.get(block) - contract = cls.get_method_contract(block.resolver, name) - if contract is None: - raise UnknownResolverMethodError( - f"Resolver {block.resolver!r} has no method {name!r}" - ) - try: - validated = contract.input_model.model_validate(arguments) - except pydantic.ValidationError as error: - raise ResolverMethodInputError.from_exception_data( - error.title, typing.cast(typing.Any, error.errors(include_url=False)) - ) from error - # Keep nested Python values typed when calling the annotated domain method; - # model_dump would convert nested input models back into dictionaries. - value = getattr(resolver, name)( - **{field: getattr(validated, field) for field in contract.input_model.model_fields} - ) - return await value if inspect.isawaitable(value) else value - - @classmethod - def match_media_type(cls, media_type: str | None) -> ResolverType | None: - """Map one specific media type to an installed exact core resolver ID. - - Evidence precedence and generic/file fallback remain extension-owned. - """ - if media_type is None: - return None - normalized = media_type.partition(";")[0].strip().lower() - if not normalized or normalized in { - "application/octet-stream", - "binary/octet-stream", - "application/binary", - }: - return None - - exact = { - "text/plain": "core.text.v1", - "text/markdown": "core.markdown.v1", - "text/html": "core.html.v1", - "application/xhtml+xml": "core.html.v1", - "application/pdf": "core.pdf.v1", - "application/epub+zip": "core.epub.v1", - "application/zip": "core.zip.v1", - "application/x-zip-compressed": "core.zip.v1", - } - resolver_id = exact.get(normalized) - if resolver_id is None: - family = normalized.partition("/")[0] - resolver_id = { - "image": "core.image.v1", - "audio": "core.audio.v1", - "video": "core.video.v1", - }.get(family) - return resolver_id if resolver_id in cls.RESOLVER_CLS else None - - -SolvedContentTV = typing.TypeVar("SolvedContentTV") -RawContentTV = typing.TypeVar("RawContentTV") -_UNSET = object() - - -class Resolver(abc.ABC, typing.Generic[SolvedContentTV, RawContentTV]): - """Resolver resolves a star graph (a block and its direct relations) - - :tparam SolvedContentTV: The type of the solved content - :tparam RawContentTV: The type of the raw content - """ - - __rsotype__: ResolverType - """Resolver type - - Extension resolvers should be namespaced and versioned, e.g., - `extensions.twitter.tweet.v1`. - """ - - draft_description: typing.ClassVar[str | None] = None - draft_input_model: typing.ClassVar[type[pydantic.BaseModel] | None] = None - - def __init_subclass__(cls, rso_type: str, **kwargs) -> None: - cls.__rsotype__ = rso_type - ResolverManager.register_resolver(cls) - return super().__init_subclass__(**kwargs) - - def __init__(self, block: BlockModel, relations: Opt[tuple[RelationModel, ...]] = None): - """Should never override __init__ in subclasses, use __post_init__ instead. - - :param block: Block to resolve. - :param relations: Relations of the block. - """ - self._block = block - self.__relations: dict[tuple[bool, bool], tuple[RelationModel, ...]] = {} - if relations is not None: - self.__relations[(True, True)] = relations - self.__solved_content: SolvedContentTV | object = _UNSET - """Solved content is the content the resolver really works with, - commonly from raw content. - """ - inline_content = ( - typing.cast(RawContentTV, self._block.content) - if self._block.storage is None - else None - ) - self.__post_init__(inline_content) - - def __post_init__(self, raw_content: Opt[RawContentTV] = None) -> None: - """Subclass post-initialization hook. - - It's suggest to set __solved_content here if possible: - ```python - async def __post_init__(self, raw_content): - if raw_content is not None: - ... # anyhow from raw_content - self.set_solved_content(solved_content) - ``` - """ - ... - - @property - def block_id(self) -> BlockID: - """Get the block ID.""" - return typing.cast(BlockID, self._block.id) - - async def get_raw_content( - self, - *, - refresh: typing.Annotated[ - bool, pydantic.Field(description="Reread current content.") - ] = False, - ) -> RawContentTV: - """Read hydrated content: text or bytes, not a storage pointer.""" - return typing.cast( - RawContentTV, - await self._block.get_hydrated_content(refresh=refresh), - ) - - async def get_transfer_url(self) -> str | None: - """Get a content transfer URL when available.""" - if self._block.storage is None: - return None - from app.business.info_base.storage import StorageManager - - storage = await StorageManager.get_storage_async(self._block.storage) - return storage.get_transfer_url(self._block.content) - - async def get_solved_content( - self, - *, - refresh: typing.Annotated[ - bool, pydantic.Field(description="Reread current content.") - ] = False, - materialize_missing: typing.Annotated[ - bool, pydantic.Field(description="Allow creation of missing derived information.") - ] = True, - ) -> SolvedContentTV: - """Read the Resolver's typed interpretation of content.""" - if refresh or self.__solved_content is _UNSET: - self.__solved_content = await self._get_solved_content( - refresh=refresh, - materialize_missing=materialize_missing, - ) - return typing.cast(SolvedContentTV, self.__solved_content) - - async def _get_solved_content( - self, - *, - refresh: bool = False, - materialize_missing: bool = True, - ) -> SolvedContentTV: - """Get the solved content (non-cache). - - Description: - When you say "will be resolved from relations", - it means you acquire the resolver of the relation's other side block, - and use its solved content. - """ - del materialize_missing - return typing.cast( - SolvedContentTV, - await self.get_raw_content(refresh=refresh), - ) - - def set_solved_content(self, content: SolvedContentTV) -> None: - self.__solved_content = content - - async def get_relations( - self, - *, - include_in: typing.Annotated[ - bool, pydantic.Field(description="Include relations pointing to this Block.") - ] = True, - include_out: typing.Annotated[ - bool, pydantic.Field(description="Include relations pointing from this Block.") - ] = True, - refresh: bool = False, - ) -> tuple[RelationModel, ...]: - """Read direct relations of this Block.""" - key = (include_in, include_out) - if refresh or key not in self.__relations: - all_relations = None if refresh else self.__relations.get((True, True)) - if all_relations is not None: - self.__relations[key] = tuple( - relation - for relation in all_relations - if (include_in and relation.to_ == self.block_id) - or (include_out and relation.from_ == self.block_id) - ) - else: - self.__relations[key] = await RelationService.get( - block_id=self.block_id, - include_in=include_in, - include_out=include_out, - ) - return self.__relations[key] - - @classmethod - # @abc.abstractmethod TODO - def create_block(cls, content, storage: Opt[StorageID] = None) -> BlockForm: ... - - @classmethod - # @abc.abstractmethod TODO - def create_graph(cls, *args, **kwargs) -> StarsGraphForm: ... - - @abc.abstractmethod - async def get_text( - self, - *, - context: typing.Annotated[ - TextProjectionContext, - pydantic.Field(description="Lexical projection is Block-local and non-recursive."), - ] = "default", - refresh: typing.Annotated[ - bool, pydantic.Field(description="Reread current content.") - ] = False, - materialize_missing: typing.Annotated[ - bool, pydantic.Field(description="Allow creation of missing derived information.") - ] = True, - ) -> str | None: - """Read a text projection; unsupported, absent and empty are distinct.""" - ... - - @abc.abstractmethod - async def get_label(self, *, refresh: bool = False) -> str: - """Read a concise label for this Block.""" - ... - - async def get_existing_async(self, blocks: BlockRepository) -> Opt[BlockModel]: - """Reconcile the exact resolver/content identity in the caller's transaction.""" - return await blocks.find_content(self._block.resolver, self._block.content) +if TYPE_CHECKING: + from inkcre_core.business.info_base.resolver.main import * diff --git a/app/business/info_base/resolver/markdown.py b/app/business/info_base/resolver/markdown.py index e381cde1..9a0aadf1 100644 --- a/app/business/info_base/resolver/markdown.py +++ b/app/business/info_base/resolver/markdown.py @@ -1,35 +1,10 @@ -"""Markdown source remains the exact text projection; rendering is a reader concern.""" +"""Legacy import; use inkcre_core for new consumers.""" -from .inspection import decode_unicode_bytes -from .label import format_label -from .main import Resolver, TextProjectionContext +from typing import TYPE_CHECKING +from importlib import import_module +import sys +sys.modules[__name__] = import_module("inkcre_core.business.info_base.resolver.markdown") -class MarkdownResolver(Resolver[str, str | bytes], rso_type="core.markdown.v1"): - async def _get_solved_content( - self, *, refresh: bool = False, materialize_missing: bool = True - ) -> str: - content = await self.get_raw_content(refresh=refresh) - return ( - content - if isinstance(content, str) - else decode_unicode_bytes(content, self.__rsotype__) - ) - - async def get_text( - self, - *, - context: TextProjectionContext = "default", - refresh: bool = False, - materialize_missing: bool = True, - ) -> str: - return await self.get_solved_content( - refresh=refresh, materialize_missing=materialize_missing - ) - - async def get_label(self, *, refresh: bool = False) -> str: - return format_label( - "markdown", - await self.get_text(refresh=refresh, materialize_missing=False), - first_line=True, - ) +if TYPE_CHECKING: + from inkcre_core.business.info_base.resolver.markdown import * diff --git a/app/business/info_base/resolver/materialization.py b/app/business/info_base/resolver/materialization.py index 1db5c82f..96c96778 100644 --- a/app/business/info_base/resolver/materialization.py +++ b/app/business/info_base/resolver/materialization.py @@ -1,76 +1,12 @@ -"""Shared mechanics for Resolver-owned faithful text derivations.""" +"""Legacy import; use inkcre_core for new consumers.""" -import logging +from typing import TYPE_CHECKING +from importlib import import_module +import sys -from app.business.ai import AIManager -from app.schemas.ai import SystemMessage, UserContentPart, UserMessage +sys.modules[__name__] = import_module( + "inkcre_core.business.info_base.resolver.materialization" +) - -logger = logging.getLogger(__name__) - - -async def generate_faithful_text( - *, - model: int, - instruction: str, - media: UserContentPart, -) -> str | None: - """Ask one exact configured Model for source-faithful text, without graph access.""" - response = await AIManager.chat( - model, - ( - SystemMessage(content=instruction), - UserMessage(content=(media,)), - ), - ) - text = (response.content or "").strip() - return text or None - - -async def materialize_text_child(block_id: int, role: str, text: str) -> None: - """Best-effort idempotent graph write for one exact information role.""" - from app.schemas.info_base.main import GraphBlockForm, GraphForm, GraphRelationForm - - from app.business.info_base.commands import persist_graph - from app.persistence.info_base.uow import graph_uow - - # Recheck after provider work in the same short scope as the graph insertion. - async with graph_uow() as uow: - if await uow.blocks.get_related(block_id, content=role) is not None: - return - await persist_graph( - GraphForm( - blocks=(GraphBlockForm(id=-1, resolver="core.text.v1", content=text),), - relations=(GraphRelationForm(from_=block_id, to_=-1, content=role),), - ), - uow, - ) - - -async def try_materialize_model_text( - *, - block_id: int, - role: str, - model: int, - instruction: str, - media: UserContentPart, -) -> None: - """Keep one unavailable derivation shallow while preserving diagnostic evidence.""" - from app.business.info_base.commands import get_related_block - - if await get_related_block(block_id, content=role) is not None: - return - try: - text = await generate_faithful_text( - model=model, - instruction=instruction, - media=media, - ) - if text is not None: - await materialize_text_child(block_id, role, text) - except Exception: - logger.warning( - "Resolver text materialization unavailable", - exc_info=True, - extra={"block": block_id, "role": role, "model": model}, - ) +if TYPE_CHECKING: + from inkcre_core.business.info_base.resolver.materialization import * diff --git a/app/business/info_base/resolver/pdf.py b/app/business/info_base/resolver/pdf.py index 9c0ea924..f7cf0c06 100644 --- a/app/business/info_base/resolver/pdf.py +++ b/app/business/info_base/resolver/pdf.py @@ -1,131 +1,10 @@ -"""Exact PDF resolver using bounded structural inspection.""" +"""Legacy import; use inkcre_core for new consumers.""" -import asyncio -from dataclasses import dataclass -from io import BytesIO +from typing import TYPE_CHECKING +from importlib import import_module +import sys -from pypdf import PdfReader -from pypdf.errors import PdfReadError +sys.modules[__name__] = import_module("inkcre_core.business.info_base.resolver.pdf") -from .contracts import ResolverContentError, TextProjectionContext -from .inspection import ( - ByteContentFacts, - detect_media_type, - format_lexical_facts, - require_bytes, -) -from .label import format_label -from .main import Resolver - - -@dataclass(frozen=True, slots=True) -class PDFSolvedContent(ByteContentFacts): - pdf_version: str | None - page_count: int | None - is_encrypted: bool - title: str | None - author: str | None - - -def _metadata_text(metadata: object, attribute: str) -> str | None: - value = getattr(metadata, attribute, None) - return value if isinstance(value, str) else None - - -def _inspect_pdf(content: bytes) -> PDFSolvedContent: - try: - reader = PdfReader(BytesIO(content), strict=True) - encrypted = reader.is_encrypted - metadata = None if encrypted else reader.metadata - page_count = None if encrypted else len(reader.pages) - except (PdfReadError, OSError, ValueError, TypeError) as error: - raise ResolverContentError("core.pdf.v1", "invalid PDF") from error - - header = reader.pdf_header - version = header.removeprefix("%PDF-") if header.startswith("%PDF-") else None - return PDFSolvedContent( - content=content, - byte_size=len(content), - detected_media_type=detect_media_type(content), - pdf_version=version, - page_count=page_count, - is_encrypted=encrypted, - title=_metadata_text(metadata, "title"), - author=_metadata_text(metadata, "author"), - ) - - -def _extract_pdf_text(content: bytes) -> str | None: - """Extract a bounded text-layer projection without treating metadata as body.""" - try: - reader = PdfReader(BytesIO(content), strict=True) - if reader.is_encrypted: - return None - parts: list[str] = [] - remaining = 1_000_000 - for page in reader.pages[:2_000]: - text = page.extract_text() or "" - if text.strip(): - selected = text[:remaining] - parts.append(selected) - remaining -= len(selected) - if remaining <= 0: - break - except (PdfReadError, OSError, ValueError, TypeError) as error: - raise ResolverContentError("core.pdf.v1", "invalid PDF text layer") from error - joined = "\n\n".join(parts).strip() - return joined or None - - -class PDFResolver( - Resolver[PDFSolvedContent, bytes], - rso_type="core.pdf.v1", -): - async def _get_solved_content( - self, - *, - refresh: bool = False, - materialize_missing: bool = True, - ) -> PDFSolvedContent: - del materialize_missing - content = require_bytes( - await self.get_raw_content(refresh=refresh), - self.__rsotype__, - ) - return await asyncio.to_thread(_inspect_pdf, content) - - async def get_text( - self, - *, - context: TextProjectionContext = "default", - refresh: bool = False, - materialize_missing: bool = True, - ) -> str | None: - del materialize_missing - content = require_bytes( - await self.get_raw_content(refresh=refresh), - self.__rsotype__, - ) - body = await asyncio.to_thread(_extract_pdf_text, content) - if context == "default": - return body - solved = await self.get_solved_content(refresh=refresh, materialize_missing=False) - metadata = format_lexical_facts( - "PDF", - ( - ("media type", solved.detected_media_type), - ("title", solved.title), - ("author", solved.author), - ("pages", solved.page_count), - ("PDF version", solved.pdf_version), - ("encrypted", solved.is_encrypted), - ), - ) - return f"{metadata}\n\n{body}" if body else metadata - - async def get_label(self, *, refresh: bool = False) -> str: - solved = await self.get_solved_content( - refresh=refresh, - materialize_missing=False, - ) - return format_label("PDF", solved.title) +if TYPE_CHECKING: + from inkcre_core.business.info_base.resolver.pdf import * diff --git a/app/business/info_base/resolver/text.py b/app/business/info_base/resolver/text.py index 47264337..100dacf0 100644 --- a/app/business/info_base/resolver/text.py +++ b/app/business/info_base/resolver/text.py @@ -1,62 +1,10 @@ -"""Exact plain-text resolver.""" +"""Legacy import; use inkcre_core for new consumers.""" -import pydantic +from typing import TYPE_CHECKING +from importlib import import_module +import sys -from .inspection import decode_unicode_bytes -from .label import format_label -from .main import Resolver, TextProjectionContext +sys.modules[__name__] = import_module("inkcre_core.business.info_base.resolver.text") - -class TextDraftInput(pydantic.BaseModel): - """Resolver-native input for drafting one plain-text Block.""" - - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - text: str - - -class TextResolver(Resolver[str, str | bytes], rso_type="core.text.v1"): - draft_description = "Create one ordinary plain-text semantic content Block." - draft_input_model = TextDraftInput - - @classmethod - def create_graph(cls, input: str | TextDraftInput): - from app.schemas.info_base.block import BlockForm - from app.schemas.info_base.main import StarsGraphForm - - text = input.text if isinstance(input, TextDraftInput) else input - return StarsGraphForm(block=BlockForm(resolver=cls.__rsotype__, content=text)) - - async def _get_solved_content( - self, - *, - refresh: bool = False, - materialize_missing: bool = True, - ) -> str: - del materialize_missing - content = await self.get_raw_content(refresh=refresh) - return ( - content - if isinstance(content, str) - else decode_unicode_bytes(content, self.__rsotype__) - ) - - async def get_text( - self, - *, - context: TextProjectionContext = "default", - refresh: bool = False, - materialize_missing: bool = True, - ) -> str: - del context - return await self.get_solved_content( - refresh=refresh, - materialize_missing=materialize_missing, - ) - - async def get_label(self, *, refresh: bool = False) -> str: - return format_label( - "text", - await self.get_text(refresh=refresh, materialize_missing=False), - first_line=True, - ) +if TYPE_CHECKING: + from inkcre_core.business.info_base.resolver.text import * diff --git a/app/business/info_base/resolver/tools.py b/app/business/info_base/resolver/tools.py index 22091973..0d66f832 100644 --- a/app/business/info_base/resolver/tools.py +++ b/app/business/info_base/resolver/tools.py @@ -1,277 +1,10 @@ -"""Agent Tool controller for typed Resolver method discovery and invocation.""" +"""Legacy import; use inkcre_core for new consumers.""" -import typing +from typing import TYPE_CHECKING +from importlib import import_module +import sys -import pydantic +sys.modules[__name__] = import_module("inkcre_core.business.info_base.resolver.tools") -from app.business.agent import AgentManager -from app.business.agent.projection import project_json -from app.business.info_base.services import BlockService -from app.schemas.ai import JSONValue -from app.schemas.info_base.block import BlockID, ResolverType -from libs.obsrv.telemetry import emit_event - -from .main import ResolverManager - - -RESOLVER_TOOL = "resolver" - - -class ResolverMethodCall(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - block_id: BlockID - method: str - arguments: dict[str, JSONValue] = pydantic.Field(default_factory=dict) - - -class ResolverDescribeInput(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - action: typing.Literal["describe"] - resolver_types: tuple[ResolverType, ...] = () - block_ids: tuple[BlockID, ...] = () - calls: tuple[ResolverMethodCall, ...] = pydantic.Field(default=(), max_length=0) - - -class ResolverInvokeInput(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - action: typing.Literal["invoke"] - resolver_types: tuple[ResolverType, ...] = pydantic.Field(default=(), max_length=0) - block_ids: tuple[BlockID, ...] = pydantic.Field(default=(), max_length=0) - calls: tuple[ResolverMethodCall, ...] = pydantic.Field(min_length=1, max_length=20) - - -class ResolverMetaToolInput( - pydantic.RootModel[ - typing.Annotated[ - ResolverDescribeInput | ResolverInvokeInput, - pydantic.Field(discriminator="action"), - ] - ] -): - pass - - -def _resolver_input_model() -> type[pydantic.BaseModel]: - common = ResolverManager.get_common_method_contracts() - variants: list[type[pydantic.BaseModel]] = [] - contracts = list(common) - common_names = {item.name for item in common} - for resolver_type in ResolverManager.RESOLVER_CLS: - contracts.extend( - contract - for contract in ResolverManager.get_method_contracts(resolver_type) - if contract.name in common_names - ) - seen: set[tuple] = set() - for contract in contracts: - signature = ( - contract.name, - tuple( - (name, repr(field.annotation), repr(field.default), repr(field.metadata)) - for name, field in contract.input_model.model_fields.items() - ), - ) - if signature in seen: - continue - seen.add(signature) - variants.append( - pydantic.create_model( - f"{contract.name}_Call_{len(variants)}", - __config__=pydantic.ConfigDict(extra="forbid"), - block_id=(int, ...), - method=( - typing.cast(typing.Any, typing.Literal)[contract.name], - pydantic.Field(description=contract.description), - ), - arguments=( - contract.input_model, - ... - if any( - field.is_required() for field in contract.input_model.model_fields.values() - ) - else pydantic.Field(default_factory=contract.input_model), - ), - ) - ) - - variants.append( - pydantic.create_model( - "ExtraMethodCall", - __base__=ResolverMethodCall, - method=( - str, - pydantic.Field( - json_schema_extra={"not": {"enum": [contract.name for contract in common]}} - ), - ), - ) - ) - call_type = typing.cast(typing.Any, typing.Union)[tuple(variants)] - invoke = pydantic.create_model( - "BoundResolverInvokeInput", - __base__=ResolverInvokeInput, - calls=(tuple[call_type, ...], pydantic.Field(min_length=1, max_length=20)), - ) - provider_envelope = pydantic.create_model( - "ResolverEnvelope", - __base__=ResolverDescribeInput, - action=(typing.Literal["describe", "invoke"], ...), - calls=(tuple[call_type, ...], pydantic.Field(default=(), max_length=20)), - ) - method_contract = pydantic.RootModel[ - typing.Annotated[ResolverDescribeInput | invoke, pydantic.Field(discriminator="action")] - ] - - # Runtime validates the dispatch envelope, then the selected Resolver validates - # each call's arguments. One invalid method argument must not reject the batch. - # The richer method schema below guides the model without changing that boundary. - class BoundResolverInput(ResolverMetaToolInput): - @classmethod - def model_json_schema(cls, *args, **kwargs) -> dict[str, typing.Any]: - method_schema = method_contract.model_json_schema(*args, **kwargs) - # Some providers infer parameter types only from top-level properties. - # Expose the wider envelope there while retaining the union's oneOf, - # discriminator and $defs so describe/invoke keep their distinct contracts. - provider_schema = provider_envelope.model_json_schema(*args, **kwargs) - method_schema.update(type="object", properties=provider_schema["properties"]) - method_schema.setdefault("$defs", {}).update(provider_schema.get("$defs", {})) - return method_schema - - return BoundResolverInput - - -@AgentManager.tool( - RESOLVER_TOOL, - input_model_factory=_resolver_input_model, - description="Describe or invoke public typed read methods on exact Block Resolvers.", -) -async def resolver(input: ResolverMetaToolInput) -> JSONValue: - request = input.root - if request.action == "describe": - found = await BlockService.get_many(request.block_ids) - resolver_ids = set(request.resolver_types) - resolver_ids.update(block.resolver for block in found) - if not request.block_ids and not request.resolver_types: - resolver_ids.update(ResolverManager.RESOLVER_CLS) - return typing.cast( - JSONValue, - { - "results": [ - { - "resolver": resolver_id, - "methods": [ - { - "name": contract.name, - "description": contract.description, - "input_schema": contract.input_schema, - } - for contract in ResolverManager.get_method_contracts(resolver_id) - ], - } - for resolver_id in sorted(resolver_ids) - if resolver_id in ResolverManager.RESOLVER_CLS - ], - "missing_blocks": sorted(set(request.block_ids) - {block.id for block in found}), - "missing_resolvers": sorted( - resolver_id - for resolver_id in resolver_ids - if resolver_id not in ResolverManager.RESOLVER_CLS - ), - }, - ) - - results: list[JSONValue] = [] - for index, call in enumerate(request.calls): - block = await BlockService.get(call.block_id) - if block is None: - results.append( - { - "index": index, - "block_id": call.block_id, - "method": call.method, - "error": "not_found", - } - ) - continue - contract = ResolverManager.get_method_contract(block.resolver, call.method) - if block.resolver not in ResolverManager.RESOLVER_CLS: - results.append( - { - "index": index, - "block_id": call.block_id, - "method": call.method, - "error": "resolver_unavailable", - "message": f"Resolver {block.resolver!r} is not registered.", - } - ) - continue - if contract is None: - results.append( - { - "index": index, - "block_id": call.block_id, - "method": call.method, - "error": "method_unavailable", - "message": "Method does not exist; use describe for available method contracts.", - "available_methods": [ - item.name for item in ResolverManager.get_method_contracts(block.resolver) - ], - } - ) - continue - try: - value = await ResolverManager.invoke_method( - block, - call.method, - call.arguments.model_dump() - if isinstance(call.arguments, pydantic.BaseModel) - else typing.cast(dict[str, typing.Any], call.arguments), - ) - projected = project_json(value) - except pydantic.ValidationError as error: - results.append( - typing.cast( - JSONValue, - { - "index": index, - "block_id": call.block_id, - "method": call.method, - "error": "invalid_arguments", - "fields": error.errors( - include_url=False, include_context=False, include_input=False - ), - "input_schema": contract.input_schema, - }, - ) - ) - except Exception as error: - results.append( - { - "index": index, - "block_id": call.block_id, - "method": call.method, - "error": type(error).__name__, - "message": str(error), - } - ) - else: - emit_event( - "inkcre.entity.read", - { - "inkcre.entity.read.kind": "resolved", - "inkcre.entity.block_ids": (call.block_id,), - "inkcre.entity.read.count": 1, - }, - ) - results.append( - { - "index": index, - "block_id": call.block_id, - "method": call.method, - "result": projected, - } - ) - return typing.cast(JSONValue, {"results": results}) +if TYPE_CHECKING: + from inkcre_core.business.info_base.resolver.tools import * diff --git a/app/business/info_base/resolver/video.py b/app/business/info_base/resolver/video.py index 93e36820..5d4ba510 100644 --- a/app/business/info_base/resolver/video.py +++ b/app/business/info_base/resolver/video.py @@ -1,201 +1,10 @@ -"""Exact video resolver using metadata-only PyAV inspection.""" +"""Legacy import; use inkcre_core for new consumers.""" -import asyncio -from dataclasses import dataclass -from fractions import Fraction -from io import BytesIO -import typing +from typing import TYPE_CHECKING +from importlib import import_module +import sys -import av -import pydantic +sys.modules[__name__] = import_module("inkcre_core.business.info_base.resolver.video") -from app.business.deployment_config import DeploymentConfigManager, DeploymentConfigService -from app.schemas.ai import VideoContentPart -from app.schemas.info_base.block import BlockForm -from app.schemas.info_base.main import StarsGraphForm - -from .audio import _container_name, _duration_ms, _primary_stream -from .contracts import ( - ResolverContentError, - TextProjectionContext, - UnsupportedResolverCapability, -) -from .inspection import ( - ByteContentFacts, - detect_media_type, - format_lexical_facts, - require_bytes, -) -from .main import Resolver -from .materialization import materialize_text_child, try_materialize_model_text - - -VIDEO_RESOLVER_CONFIG_KEY = "core.resolver.video" -VIDEO_RESOLVER_CONFIG_SCHEMA = "core.resolver.video.config.v1" - - -class VideoResolverConfig(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - text_model: int - transcript_model: int - - -DeploymentConfigManager.register_schema( - VIDEO_RESOLVER_CONFIG_SCHEMA, VideoResolverConfig, keys=(VIDEO_RESOLVER_CONFIG_KEY,) -) - - -@dataclass(frozen=True, slots=True) -class VideoSolvedContent(ByteContentFacts): - container: str | None - video_codec: str | None - duration_ms: int | None - width: int | None - height: int | None - frame_rate: float | None - subtitles: tuple[str, ...] - - -def _frame_rate(rate: Fraction | None) -> float | None: - return float(rate) if rate is not None else None - - -def _subtitles(container: typing.Any) -> tuple[str, ...]: - values: list[str] = [] - streams = tuple(container.streams.subtitles) - if not streams: - return () - for packet in container.demux(*streams): - try: - decoded = packet.decode() - except (av.FFmpegError, OSError, ValueError): - continue - for subtitle in decoded: - raw = getattr(subtitle, "dialogue", None) or getattr(subtitle, "text", None) - if isinstance(raw, bytes): - text = raw.decode("utf-8", errors="replace").strip() - elif isinstance(raw, str): - text = raw.strip() - else: - text = "" - if text and (not values or values[-1] != text): - values.append(text) - return tuple(values) - - -def _inspect_video(content: bytes) -> VideoSolvedContent: - try: - with av.open(BytesIO(content), mode="r") as container: - stream = _primary_stream(container.streams.video) - if stream is None: - raise ResolverContentError("core.video.v1", "no video stream") - codec = stream.codec_context - facts = VideoSolvedContent( - content=content, - byte_size=len(content), - detected_media_type=detect_media_type(content), - container=_container_name(container.format.name), - video_codec=codec.name, - duration_ms=_duration_ms(container, stream), - width=codec.width, - height=codec.height, - frame_rate=_frame_rate(stream.average_rate), - subtitles=_subtitles(container), - ) - except ResolverContentError: - raise - except (av.FFmpegError, OSError, ValueError) as error: - raise ResolverContentError("core.video.v1", "invalid video") from error - return facts - - -class VideoResolver( - Resolver[VideoSolvedContent, bytes], - rso_type="core.video.v1", -): - @classmethod - def create_graph(cls, url: str) -> StarsGraphForm: - return StarsGraphForm( - block=BlockForm(resolver=cls.__rsotype__, content=url, storage=-1) - ) - - async def _get_solved_content( - self, - *, - refresh: bool = False, - materialize_missing: bool = True, - ) -> VideoSolvedContent: - del materialize_missing - content = require_bytes( - await self.get_raw_content(refresh=refresh), - self.__rsotype__, - ) - return await asyncio.to_thread(_inspect_video, content) - - async def get_text( - self, - *, - context: TextProjectionContext = "default", - refresh: bool = False, - materialize_missing: bool = True, - ) -> str: - if context == "default": - raise UnsupportedResolverCapability(self.__rsotype__, "text") - solved = await self.get_solved_content(refresh=refresh, materialize_missing=False) - if materialize_missing: - if solved.subtitles: - await materialize_text_child( - self.block_id, "subtitle", "\n\n".join(solved.subtitles) - ) - config = typing.cast( - VideoResolverConfig | None, - await DeploymentConfigService.get(VIDEO_RESOLVER_CONFIG_KEY), - ) - if config is not None: - media_type = solved.detected_media_type or ( - f"video/{solved.container}" if solved.container is not None else None - ) - if media_type is not None: - media = VideoContentPart( - data=solved.content, - mime_type=media_type, - transfer_url=await self.get_transfer_url(), - ) - await try_materialize_model_text( - block_id=self.block_id, - role="transcript", - model=config.transcript_model, - instruction=( - "Transcribe only speech and authored spoken language in the video. " - "Return plain text with no summary or visual description." - ), - media=media, - ) - await try_materialize_model_text( - block_id=self.block_id, - role="text", - model=config.text_model, - instruction=( - "Transcribe only written text visibly present in the video. " - "Return plain text with no summary or scene description." - ), - media=media, - ) - return format_lexical_facts( - "video", - ( - ("media type", solved.detected_media_type), - ("container", solved.container), - ("codec", solved.video_codec), - ("duration ms", solved.duration_ms), - ("width", solved.width), - ("height", solved.height), - ("frame rate", solved.frame_rate), - ("subtitle cues", len(solved.subtitles) or None), - ), - ) - - async def get_label(self, *, refresh: bool = False) -> str: - del refresh - return "video" +if TYPE_CHECKING: + from inkcre_core.business.info_base.resolver.video import * diff --git a/app/business/info_base/resolver/zip.py b/app/business/info_base/resolver/zip.py index 0a0a0f35..472bf2d2 100644 --- a/app/business/info_base/resolver/zip.py +++ b/app/business/info_base/resolver/zip.py @@ -1,86 +1,10 @@ -"""Exact ZIP resolver using central-directory metadata only.""" +"""Legacy import; use inkcre_core for new consumers.""" -import asyncio -from dataclasses import dataclass -from io import BytesIO -import zipfile +from typing import TYPE_CHECKING +from importlib import import_module +import sys -from .contracts import ( - ResolverContentError, - TextProjectionContext, - UnsupportedResolverCapability, -) -from .inspection import ByteContentFacts, format_lexical_facts, require_bytes -from .main import Resolver +sys.modules[__name__] = import_module("inkcre_core.business.info_base.resolver.zip") - -@dataclass(frozen=True, slots=True) -class ZIPSolvedContent(ByteContentFacts): - member_count: int - total_compressed_bytes: int - total_uncompressed_bytes: int - compression_methods: tuple[int, ...] - encrypted_member_count: int - - -def _inspect_zip(content: bytes) -> ZIPSolvedContent: - try: - with zipfile.ZipFile(BytesIO(content)) as archive: - members = archive.infolist() - except (zipfile.BadZipFile, OSError, ValueError) as error: - raise ResolverContentError("core.zip.v1", "invalid ZIP archive") from error - - return ZIPSolvedContent( - content=content, - byte_size=len(content), - detected_media_type="application/zip", - member_count=len(members), - total_compressed_bytes=sum(member.compress_size for member in members), - total_uncompressed_bytes=sum(member.file_size for member in members), - compression_methods=tuple(sorted({member.compress_type for member in members})), - encrypted_member_count=sum(bool(member.flag_bits & 0x1) for member in members), - ) - - -class ZIPResolver( - Resolver[ZIPSolvedContent, bytes], - rso_type="core.zip.v1", -): - async def _get_solved_content( - self, - *, - refresh: bool = False, - materialize_missing: bool = True, - ) -> ZIPSolvedContent: - del materialize_missing - content = require_bytes( - await self.get_raw_content(refresh=refresh), - self.__rsotype__, - ) - return await asyncio.to_thread(_inspect_zip, content) - - async def get_text( - self, - *, - context: TextProjectionContext = "default", - refresh: bool = False, - materialize_missing: bool = True, - ) -> str: - if context == "default": - raise UnsupportedResolverCapability(self.__rsotype__, "text") - del materialize_missing - solved = await self.get_solved_content(refresh=refresh, materialize_missing=False) - return format_lexical_facts( - "ZIP", - ( - ("media type", solved.detected_media_type), - ("members", solved.member_count), - ("compressed bytes", solved.total_compressed_bytes), - ("uncompressed bytes", solved.total_uncompressed_bytes), - ("encrypted members", solved.encrypted_member_count), - ), - ) - - async def get_label(self, *, refresh: bool = False) -> str: - del refresh - return "ZIP" +if TYPE_CHECKING: + from inkcre_core.business.info_base.resolver.zip import * diff --git a/app/business/info_base/services.py b/app/business/info_base/services.py index 913da8e9..4e258a7d 100644 --- a/app/business/info_base/services.py +++ b/app/business/info_base/services.py @@ -1,183 +1,10 @@ -"""Independent info-base use cases; composed writes use GraphUnitOfWork directly.""" +"""Legacy import; use inkcre_core for new consumers.""" -import typing -from typing import Optional as Opt -from app.schemas.info_base.block import BlockForm, BlockModel, BlockID, ResolverType -from app.schemas.info_base.relation import RelationModel, RelationID -from app.schemas.info_base.storage import StorageID -from utils.types_ import Undefined, _undefined -from app.persistence.info_base.uow import graph_uow +from typing import TYPE_CHECKING +from importlib import import_module +import sys +sys.modules[__name__] = import_module("inkcre_core.business.info_base.services") -class BlockService: - @staticmethod - async def get_many(block_ids: typing.Collection[BlockID]) -> tuple[BlockModel, ...]: - async with graph_uow() as uow: - result = await uow.blocks.get_many(block_ids) - return result - - @staticmethod - async def get_random_many(count: int) -> tuple[BlockModel, ...]: - async with graph_uow() as uow: - result = await uow.blocks.get_random_many(count) - return result - - @staticmethod - async def get_random() -> Opt[BlockModel]: - async with graph_uow() as uow: - result = await uow.blocks.get_random() - return result - - @staticmethod - async def get(block_id: BlockID) -> Opt[BlockModel]: - async with graph_uow() as uow: - result = await uow.blocks.get(block_id) - return result - - @staticmethod - async def edit_block( - block_id: BlockID, - content: Opt[str] = None, - resolver: Opt[ResolverType] = None, - storage: Opt[StorageID] | Undefined = _undefined, - ) -> BlockModel: - async with graph_uow() as uow: - result = await uow.blocks.edit_block(block_id, content, resolver, storage) - return result - - @staticmethod - async def delete(block_id: BlockID) -> bool: - async with graph_uow() as uow: - result = await uow.blocks.delete(block_id) - return result - - @staticmethod - async def get_recent( - num: int = 10, resolver: Opt[ResolverType] = None - ) -> tuple[BlockModel, ...]: - """获取最新的块 - - 按创建时间倒序排序。 - - :param num: 获取的块数量 - :param resolver: 限定解析器类型, None则不限定 - """ - async with graph_uow() as uow: - result = await uow.blocks.get_recent(num, resolver) - return result - - @staticmethod - async def create(form: BlockForm) -> BlockModel: - async with graph_uow() as uow: - result = await uow.blocks.create(form) - return result - - -class RelationService: - @staticmethod - async def get_text( - relation: RelationModel, - *, - refresh: bool = False, - ) -> str | None: - """Project one directed dynamic property through Block-local endpoint labels.""" - if not relation.content.strip(): - return None - async with graph_uow() as uow: - endpoints = { - block.id: block - for block in await uow.blocks.get_many((relation.from_, relation.to_)) - } - from_block = endpoints.get(relation.from_) - to_block = endpoints.get(relation.to_) - if from_block is None or to_block is None: - return None - - # Local imports avoid reversing Resolver -> RelationService ownership. - from app.business.info_base.resolver import ResolverManager - - subject = await ResolverManager.get(from_block).get_label(refresh=refresh) - value = await ResolverManager.get(to_block).get_label(refresh=refresh) - if not subject.strip() or not value.strip(): - return None - return f"subject:\n{subject}\nproperty:\n{relation.content}\nvalue:\n{value}" - - @staticmethod - async def get_many( - relation_ids: typing.Collection[RelationID], - ) -> tuple[RelationModel, ...]: - async with graph_uow() as uow: - result = await uow.relations.get_many(relation_ids) - return result - - @staticmethod - async def get_by_id(relation_id: RelationID) -> RelationModel | None: - async with graph_uow() as uow: - result = await uow.relations.get_by_id(relation_id) - return result - - @staticmethod - async def get_endpoint_page( - block_ids: typing.Collection[BlockID], - *, - endpoint: typing.Literal["from", "to"], - contents: typing.Collection[str] = (), - cursor: RelationID | None = None, - limit: int = 21, - ) -> tuple[RelationModel, ...]: - async with graph_uow() as uow: - result = await uow.relations.get_endpoint_page( - block_ids, endpoint=endpoint, contents=contents, cursor=cursor, limit=limit - ) - return result - - @staticmethod - async def get( - block_id: BlockID, - include_in: bool = True, - include_out: bool = True, - content: Opt[str] = None, - ) -> tuple[RelationModel, ...]: - async with graph_uow() as uow: - result = await uow.relations.get(block_id, include_in, include_out, content) - return result - - @staticmethod - async def update( - relation_id: RelationID, - *, - from_: BlockID | Undefined = _undefined, - to_: BlockID | Undefined = _undefined, - content: str | Undefined = _undefined, - ) -> RelationModel: - async with graph_uow() as uow: - result = await uow.relations.update( - relation_id, from_=from_, to_=to_, content=content - ) - return result - - @staticmethod - async def delete(relation_id: RelationID) -> bool: - async with graph_uow() as uow: - result = await uow.relations.delete(relation_id) - return result - - @staticmethod - async def create(from_: BlockID, to_: BlockID, content: str) -> RelationModel: - async with graph_uow() as uow: - result = await uow.relations.create(from_, to_, content) - return result - - -async def get_entity_records( - block_ids: typing.Collection[BlockID], relation_ids: typing.Collection[RelationID] -) -> tuple[dict[BlockID, BlockModel], dict[RelationID, RelationModel]]: - """Read both entity kinds in one short scope before any content hydration.""" - async with graph_uow() as uow: - blocks = await uow.blocks.get_many(block_ids) - relations = await uow.relations.get_many(relation_ids) - # Persisted rows always have primary keys; the shared model also permits new rows. - return ( - {typing.cast(BlockID, block.id): block for block in blocks}, - {typing.cast(RelationID, row.id): row for row in relations}, - ) +if TYPE_CHECKING: + from inkcre_core.business.info_base.services import * diff --git a/app/business/info_base/storage/__init__.py b/app/business/info_base/storage/__init__.py index 2f318982..f53fbbe9 100644 --- a/app/business/info_base/storage/__init__.py +++ b/app/business/info_base/storage/__init__.py @@ -1,15 +1,3 @@ -from .main import Storage, StorageManager, WritableStorage +"""Legacy import; use inkcre_core for new consumers.""" -__all__ = [ - "Storage", - "StorageManager", - "WritableStorage", - "HTTPStorage", - "HTTPStorageConfig", - "StorageContentTooLargeError", - "PostgreSQLBinaryStorage", -] - -# Import storage implementations to ensure they're registered -from .http import HTTPStorage, HTTPStorageConfig, StorageContentTooLargeError -from .postgresql import PostgreSQLBinaryStorage +from inkcre_core.business.info_base.storage import * diff --git a/app/business/info_base/storage/http.py b/app/business/info_base/storage/http.py index b86143c7..65b01158 100644 --- a/app/business/info_base/storage/http.py +++ b/app/business/info_base/storage/http.py @@ -1,86 +1,10 @@ -"""Mechanics-only HTTP storage returning bounded response bytes.""" +"""Legacy import; use inkcre_core for new consumers.""" -from __future__ import annotations +from typing import TYPE_CHECKING +from importlib import import_module +import sys -import aiohttp -import pydantic -import sqlmodel +sys.modules[__name__] = import_module("inkcre_core.business.info_base.storage.http") -from utils.base import AIOHTTP_CONNECTOR_GETTER - -from .main import Storage - - -__all__ = [ - "HTTPStorageConfig", - "HTTPStorage", - "StorageContentTooLargeError", -] - - -DEFAULT_MAX_RESPONSE_BYTES = 64 * 1024 * 1024 -READ_CHUNK_BYTES = 64 * 1024 - - -class HTTPStorageConfig(sqlmodel.SQLModel): - """Configuration for bounded HTTP byte retrieval.""" - - timeout: int = pydantic.Field(default=30, gt=0) - follow_redirects: bool = True - max_response_bytes: int = pydantic.Field( - default=DEFAULT_MAX_RESPONSE_BYTES, - gt=0, - ) - - -class StorageContentTooLargeError(ValueError): - """The configured byte boundary was exceeded while hydrating content.""" - - -class HTTPStorage( - Storage[HTTPStorageConfig, bytes], - stg_type="http", - config_cls=HTTPStorageConfig, -): - """Fetch opaque bytes over HTTP(S) without assigning content semantics.""" - - @staticmethod - def _url(block_content: str) -> str: - url = block_content.strip() - if not url.lower().startswith(("http://", "https://")): - raise ValueError("HTTP storage pointer must use http:// or https://") - return url - - def get_transfer_url(self, block_content: str) -> str: - return self._url(block_content) - - async def get_raw_content(self, block_content: str) -> bytes: - url = self._url(block_content) - - timeout = aiohttp.ClientTimeout(total=self._config.timeout) - async with aiohttp.ClientSession( - connector=AIOHTTP_CONNECTOR_GETTER(), - timeout=timeout, - ) as session: - async with session.get( - url, - allow_redirects=self._config.follow_redirects, - ) as response: - response.raise_for_status() - content_length = response.content_length - if content_length is not None and content_length > self._config.max_response_bytes: - raise StorageContentTooLargeError( - f"HTTP response declares {content_length} bytes; " - f"limit is {self._config.max_response_bytes}" - ) - - chunks: list[bytes] = [] - received = 0 - async for chunk in response.content.iter_chunked(READ_CHUNK_BYTES): - received += len(chunk) - if received > self._config.max_response_bytes: - raise StorageContentTooLargeError( - f"HTTP response exceeded {self._config.max_response_bytes} bytes" - ) - chunks.append(chunk) - return b"".join(chunks) +if TYPE_CHECKING: + from inkcre_core.business.info_base.storage.http import * diff --git a/app/business/info_base/storage/main.py b/app/business/info_base/storage/main.py index 59d45389..2c36ea82 100644 --- a/app/business/info_base/storage/main.py +++ b/app/business/info_base/storage/main.py @@ -1,183 +1,10 @@ -__all__ = [ - "StorageManager", - "Storage", -] +"""Legacy import; use inkcre_core for new consumers.""" -import abc -import importlib -import sqlmodel -import typing -from app.persistence.info_base.storage import StorageRepository -from app.database_contract.profile import BUILTIN_STORAGES, BUILTIN_STORAGE_TYPES_BY_ID -from app.schemas.info_base.storage import ( - StorageID, - StorageTypeID, - StorageModel, -) +from typing import TYPE_CHECKING +from importlib import import_module +import sys +sys.modules[__name__] = import_module("inkcre_core.business.info_base.storage.main") -ConfigTV = typing.TypeVar("ConfigTV", bound=sqlmodel.SQLModel) -ContentTV = typing.TypeVar("ContentTV") - - -class _EmptyConfig(sqlmodel.SQLModel): - """Default empty config class for storages without configuration.""" - - -class StorageManager: - _STORAGE_CLASSES: dict[StorageTypeID, type["Storage"]] = {} - """Storage type registry: type ID -> Storage class""" - - @classmethod - def register_storage(cls, storage_cls: type["Storage"]): - """Register a storage class in memory without external side effects.""" - cls._STORAGE_CLASSES[storage_cls.__stgtype__] = storage_cls - - @classmethod - async def get_storage_async(cls, storage_id: StorageID) -> "Storage": - from app.persistence.info_base.uow import graph_uow - - async with graph_uow() as uow: - record = await uow.storage.get(storage_id) - return cls.from_record(record) - - @classmethod - def from_record(cls, record: StorageModel) -> "Storage": - storage_class = cls._STORAGE_CLASSES.get(record.type) - if storage_class is None: - module_path, class_name = record.type.rsplit(".", 1) - storage_class = getattr(importlib.import_module(module_path), class_name) - return storage_class(record) - - @classmethod - async def setup_builtin_storages_async(cls) -> None: - from app.persistence.info_base.uow import graph_uow - - records = [] - for storage_cls in cls._STORAGE_CLASSES.values(): - builtin = BUILTIN_STORAGE_TYPES_BY_ID.get(storage_cls.__stgtype__) - records.append( - { - "id": storage_cls.__stgtype__, - "description": builtin.description - if builtin - else storage_cls.__doc__ or "No description.", - "config_schema": builtin.config_schema - if builtin - else storage_cls.__configschema__, - "writable": issubclass(storage_cls, WritableStorage), - } - ) - async with graph_uow() as uow: - await uow.storage.sync_types(records) - await uow.storage.sync_builtins( - [ - {"id": row.id, "type": row.type, "nickname": row.nickname, "config": row.config} - for row in BUILTIN_STORAGES - ] - ) - - -class Storage(abc.ABC, typing.Generic[ConfigTV, ContentTV]): - """Storage base. - Storage retrieves the raw ("real") content from block record. - - Generic parameters: - - ConfigTV: Configuration type variable - - ContentTV: Content type variable - """ - - __configschema__: dict - """Storage configuration JSON schema""" - __configcls__: type[ConfigTV] - __stgtype__: StorageTypeID - """Storage type identifier""" - - def __init_subclass__( - cls, - stg_type: StorageTypeID | None = None, - config_cls: type[ConfigTV] = _EmptyConfig, - **kwargs, - ) -> None: - """ - :param stg_type: Unique storage type string - :param config_cls: Configuration class for the storage - """ - # ConfigTV is bound by the concrete storage subclass. - if stg_type is not None: - cls.__configcls__ = config_cls # pyrefly: ignore[no-access] - cls.__configschema__ = config_cls.model_json_schema() - cls.__stgtype__ = stg_type - StorageManager.register_storage(cls) - return super().__init_subclass__(**kwargs) - - def __init__(self, storage_record: StorageModel): - if storage_record.id is None: - raise ValueError("Storage must be persisted before use") - self._id = storage_record.id - self._config = self.__configcls__.model_validate(storage_record.config) - - self.__post_init__() - - @property - def storage_id(self) -> StorageID: - """Return the persisted Storage reference represented by this instance.""" - return self._id - - def __post_init__(self): - """Post-initialization hook for subclasses.""" - pass - - async def get_raw_content(self, block_content: str) -> ContentTV: - """Get the raw content of the block.""" - raise NotImplementedError( - f"{self.__class__.__name__}.get_raw_content() must be implemented by subclasses." - ) - - def get_transfer_url(self, block_content: str) -> str | None: - """Return an optional external byte-transfer hint for AI/provider use. - - The URL is neither content authority nor a promise that every external - provider can fetch it. Callers must retain hydrated content as the primary - value and treat this result as an optimization hint. - """ - del block_content - return None - - -class WritableStorage(Storage[ConfigTV, ContentTV], abc.ABC): - """Storage capability for raw content owned by the current deployment.""" - - async def get_raw_content(self, block_content: str) -> ContentTV: - from app.persistence.info_base.uow import graph_uow - - async with graph_uow() as uow: - return await self.read_content(block_content, uow.storage) - - async def create_content(self, content: ContentTV, storage: StorageRepository) -> str: - return self.serialize_pointer(await self.write_content(content, storage)) - - @abc.abstractmethod - async def read_content( - self, block_content: str, storage: StorageRepository - ) -> ContentTV: ... - - @abc.abstractmethod - async def write_content( - self, content: ContentTV, storage: StorageRepository - ) -> typing.Any: ... - - @abc.abstractmethod - async def update_content( - self, block_content: str, content: ContentTV, storage: StorageRepository - ) -> bool: ... - - @abc.abstractmethod - async def delete_content( - self, block_content: str, storage: StorageRepository - ) -> bool: ... - - @abc.abstractmethod - def serialize_pointer(self, pointer: typing.Any) -> str: - """Encode a storage-owned key as one opaque block pointer string.""" - ... +if TYPE_CHECKING: + from inkcre_core.business.info_base.storage.main import * diff --git a/app/business/info_base/storage/postgresql.py b/app/business/info_base/storage/postgresql.py index 26ee5826..c6d2827f 100644 --- a/app/business/info_base/storage/postgresql.py +++ b/app/business/info_base/storage/postgresql.py @@ -1,59 +1,10 @@ -"""Writable PostgreSQL binary storage.""" +"""Legacy import; use inkcre_core for new consumers.""" -import uuid +from typing import TYPE_CHECKING +from importlib import import_module +import sys -import pydantic -import sqlmodel +sys.modules[__name__] = import_module("inkcre_core.business.info_base.storage.postgresql") -from .main import WritableStorage -from app.persistence.info_base.storage import StorageRepository - - -class PostgreSQLBinaryStorageConfig(sqlmodel.SQLModel): - pass - - -class PostgreSQLBlobPointer(pydantic.BaseModel): - """Minimum pointer fields accepted from storage-backed block content.""" - - model_config = pydantic.ConfigDict(extra="ignore") - - blob_id: uuid.UUID - - -class StorageBlobNotFoundError(LookupError): - pass - - -class PostgreSQLBinaryStorage( - WritableStorage[PostgreSQLBinaryStorageConfig, bytes], - stg_type="postgresql_binary", - config_cls=PostgreSQLBinaryStorageConfig, -): - """Store raw bytes in the protocol database and return an opaque UUID pointer.""" - - def serialize_pointer(self, pointer: object) -> str: - blob_id = pydantic.TypeAdapter(uuid.UUID).validate_python(pointer) - return PostgreSQLBlobPointer(blob_id=blob_id).model_dump_json() - - async def read_content( - self, block_content: str, storage: StorageRepository, *, lock: bool = False - ) -> bytes: - pointer = PostgreSQLBlobPointer.model_validate_json(block_content) - content = await storage.read_blob(pointer.blob_id, lock=lock) - if content is None: - raise StorageBlobNotFoundError(f"Storage blob {pointer.blob_id} not found") - return content - - async def write_content(self, content: bytes, storage: StorageRepository) -> uuid.UUID: - return await storage.create_blob(content) - - async def update_content( - self, block_content: str, content: bytes, storage: StorageRepository - ) -> bool: - pointer = PostgreSQLBlobPointer.model_validate_json(block_content) - return await storage.update_blob(pointer.blob_id, content) - - async def delete_content(self, block_content: str, storage: StorageRepository) -> bool: - pointer = PostgreSQLBlobPointer.model_validate_json(block_content) - return await storage.delete_blob(pointer.blob_id) +if TYPE_CHECKING: + from inkcre_core.business.info_base.storage.postgresql import * diff --git a/app/business/info_base/tools.py b/app/business/info_base/tools.py index a1dbdb0f..cbd22d81 100644 --- a/app/business/info_base/tools.py +++ b/app/business/info_base/tools.py @@ -1,152 +1,10 @@ -"""Agent Tool controllers for info-base retrieval and entity reads.""" +"""Legacy import; use inkcre_core for new consumers.""" -import typing +from typing import TYPE_CHECKING +from importlib import import_module +import sys -import pydantic +sys.modules[__name__] = import_module("inkcre_core.business.info_base.tools") -from app.business.agent import AgentManager -from app.business.agent.projection import project_json -from app.schemas.ai import JSONValue -from app.schemas.info_base.block import BlockModel -from app.schemas.info_base.relation import RelationModel -from app.schemas.lexical_retrieval import LexicalRetrievalResult -from libs.obsrv.telemetry import emit_event - -from .main import InfoBaseManager - - -RETRIEVE_TOOL = "retrieve" -GET_ENTITIES_TOOL = "get_entities" -RetrievalMode: typing.TypeAlias = typing.Literal["lexical", "semantic", "hybrid"] - - -class RetrieveInput(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - query: str - mode: RetrievalMode = "hybrid" - limit: int = pydantic.Field(default=20, ge=1, le=20) - - @pydantic.field_validator("query") - @classmethod - def non_empty_query(cls, value: str) -> str: - if not value.strip(): - raise ValueError("query must not be empty") - return value - - -class EntityReference(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - type: typing.Literal["block", "relation"] - id: int - - -class GetEntitiesInput(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - entities: tuple[EntityReference, ...] = pydantic.Field(default=(), max_length=20) - random_count: int = pydantic.Field(default=1, ge=1, le=20) - - @pydantic.model_validator(mode="after") - def validate_selection(self) -> typing.Self: - if self.entities and self.random_count != 1: - raise ValueError("random_count only applies when entities is empty") - return self - - -@AgentManager.tool( - RETRIEVE_TOOL, - description="Retrieve lexical, semantic, or separate hybrid results for one query.", -) -async def retrieve(input: RetrieveInput) -> JSONValue: - branches = await InfoBaseManager.retrieve(input.query, input.mode, input.limit) - payload: dict[str, JSONValue] = {} - for name, branch in branches.items(): - if branch.error is not None: - payload[name] = { - "error": type(branch.error).__name__, - "message": str(branch.error), - } - continue - result = branch.result - assert result is not None - if isinstance(result, LexicalRetrievalResult): - block_ids = tuple( - match.block.id for match in result.matches if match.block.id is not None - ) - relation_ids: tuple[int, ...] = () - else: - block_ids = tuple( - match.entity.id - for match in result.matches - if match.type == "block" and match.entity.id is not None - ) - relation_ids = tuple( - match.entity.id - for match in result.matches - if match.type == "relation" and match.entity.id is not None - ) - emit_event( - "inkcre.retrieval.candidates", - { - "inkcre.retrieval.method": "lexical" if name == "lexical" else "semantic", - "inkcre.entity.block_ids": block_ids, - "inkcre.entity.relation_ids": relation_ids, - "inkcre.retrieval.candidate_count": len(result.matches), - }, - ) - if name == "lexical": - payload[name] = { - "matches": [ - { - "entity": {"entity_type": "block", "entity_id": match.block.id}, - **match.model_dump(mode="json", exclude={"block"}), - } - for match in typing.cast(typing.Any, result).matches - ] - } - else: - payload[name] = { - **result.model_dump(mode="json", exclude={"matches"}), - "matches": [ - { - "entity": {"entity_type": match.type, "entity_id": match.entity.id}, - "score": match.score, - } - for match in typing.cast(typing.Any, result).matches - ], - } - return payload - - -@AgentManager.tool( - GET_ENTITIES_TOOL, - description=( - "Read persisted Blocks or Relations without resolving content. " - "Null may indicate an incorrect entity type." - ), -) -async def get_entities(input: GetEntitiesInput) -> JSONValue: - result = await InfoBaseManager.get_entities( - tuple((reference.type, reference.id) for reference in input.entities), - random_count=input.random_count, - ) - emit_event( - "inkcre.entity.read", - { - "inkcre.entity.read.kind": "persisted", - "inkcre.entity.block_ids": tuple( - entity.id - for entity in result - if isinstance(entity, BlockModel) and entity.id is not None - ), - "inkcre.entity.relation_ids": tuple( - entity.id - for entity in result - if isinstance(entity, RelationModel) and entity.id is not None - ), - "inkcre.entity.read.count": sum(entity is not None for entity in result), - }, - ) - return project_json(result) +if TYPE_CHECKING: + from inkcre_core.business.info_base.tools import * diff --git a/app/business/job.py b/app/business/job.py index a85d1bbc..a804caa0 100644 --- a/app/business/job.py +++ b/app/business/job.py @@ -1,402 +1,10 @@ -"""Deep runtime for typed durable background Jobs.""" +"""Legacy import; use inkcre_core for new consumers.""" -import abc -import asyncio -import dataclasses -import typing +from typing import TYPE_CHECKING +from importlib import import_module +import sys -import jsonschema # pyrefly: ignore[untyped-import] -import pydantic +sys.modules[__name__] = import_module("inkcre_core.business.job") -from app.database_contract.profile import BUILTIN_JOB_TYPES_BY_ID -from app.persistence.job.uow import JobUnitOfWork, job_uow -from app.scheduler import scheduler, with_trace_id -from app.schemas.job import JobID, JobModel, JobStatus, JobTypeID, JobTypeModel -from libs.obsrv.main import get_logger -from libs.obsrv.telemetry import emit_event, operation -from app.observability import submission_context, submission_links -from opentelemetry.context import Context - - -LOGGER = get_logger().getChild(__name__) -ParametersTV = typing.TypeVar("ParametersTV", bound=pydantic.BaseModel) - - -class UnknownJobTypeError(ValueError): - """A submitted exact Job type is absent from the deployment catalog.""" - - -@dataclasses.dataclass -class _Execution: - task: asyncio.Task - cancellation_requested: bool = False - closing: bool = False - - def cancel(self) -> None: - # Do not interrupt cleanup already started by timeout or an earlier abort. - if not self.cancellation_requested and not self.closing: - self.cancellation_requested = True - if not self.task.cancelling(): - self.task.cancel() - - -class JobHandler(abc.ABC, typing.Generic[ParametersTV]): - """One exact executable Job contract registered by its owning module.""" - - type: typing.ClassVar[JobTypeID] - description: typing.ClassVar[str] - parameters_model: typing.ClassVar[type[pydantic.BaseModel]] - default_timeout_seconds: typing.ClassVar[int] - - def __init_subclass__( - cls, - *, - job_type: JobTypeID, - description: str, - parameters_model: type[ParametersTV], - default_timeout_seconds: int, - **kwargs, - ) -> None: - if default_timeout_seconds <= 0: - raise ValueError("default_timeout_seconds must be positive") - cls.type = job_type - cls.description = description - cls.parameters_model = parameters_model - cls.default_timeout_seconds = default_timeout_seconds - JobManager.register_handler(cls) - super().__init_subclass__(**kwargs) - - @classmethod - def validate_parameters(cls, parameters: dict[str, typing.Any]) -> ParametersTV: - return typing.cast(ParametersTV, cls.parameters_model.model_validate(parameters)) - - @classmethod - async def normalize_parameters( - cls, parameters: dict[str, typing.Any], uow: JobUnitOfWork - ) -> dict[str, typing.Any]: - """Validate a new Job/Cron input; owners may resolve nested catalog contracts. - - Execution restores the persisted model through validate_parameters instead; - it does not repeat submission-only normalization. - """ - del uow - return cls.validate_parameters(parameters).model_dump(mode="json") - - @classmethod - @abc.abstractmethod - async def can_handle(cls, parameters: ParametersTV) -> bool: - """Return whether this runtime can execute these parameters now.""" - ... - - @classmethod - @abc.abstractmethod - async def handle(cls, job: JobModel, parameters: ParametersTV) -> None: - """Execute one already claimed Job.""" - ... - - -class JobManager: - """Own Job Handler registration, typed creation, claim and terminal closure.""" - - _handlers: dict[JobTypeID, type[JobHandler]] = {} - _active: dict[JobID, _Execution] = {} - _accepting = True - - @classmethod - def register_handler(cls, handler: type[JobHandler]) -> None: - existing = cls._handlers.get(handler.type) - if existing is handler: - return - if existing is not None: - raise ValueError( - f"Job type {handler.type!r} is already handled by " - f"{existing.__module__}.{existing.__qualname__}" - ) - cls._handlers[handler.type] = handler - - @classmethod - async def sync_job_types(cls) -> None: - rows = [] - for handler in cls._handlers.values(): - builtin = BUILTIN_JOB_TYPES_BY_ID.get(handler.type) - rows.append( - dict( - id=handler.type, - description=builtin.description if builtin is not None else handler.description, - parameters_schema=( - builtin.parameters_schema - if builtin is not None - else handler.parameters_model.model_json_schema() - ), - default_timeout_seconds=( - builtin.default_timeout_seconds - if builtin is not None - else handler.default_timeout_seconds - ), - ) - ) - async with job_uow() as uow: - await uow.jobs.sync_types(rows) - - @classmethod - async def normalize_parameters( - cls, - job_type: JobTypeID, - parameters: dict[str, typing.Any], - uow: JobUnitOfWork, - ) -> dict[str, typing.Any]: - handler = cls._handlers.get(job_type) - if handler is not None: - return await handler.normalize_parameters(parameters, uow) - - persisted_type = await uow.jobs.get_type(job_type) - if persisted_type is None: - raise UnknownJobTypeError(f"Unknown Job type: {job_type}") - jsonschema.Draft202012Validator(persisted_type.parameters_schema).validate(parameters) - return parameters - - @classmethod - async def create( - cls, - job_type: JobTypeID, - parameters: dict[str, typing.Any], - timeout_seconds: int | None = None, - ) -> JobModel: - """Validate and persist one independent pending Job.""" - async with job_uow() as uow: - job = await cls.create_in_uow(job_type, parameters, timeout_seconds, uow=uow) - emit_event("job.submitted", {"inkcre.job.id": job.id or 0}) - return job - - @classmethod - async def create_in_uow( - cls, - job_type: JobTypeID, - parameters: dict[str, typing.Any], - timeout_seconds: int | None = None, - *, - uow: JobUnitOfWork, - ) -> JobModel: - normalized = await cls.normalize_parameters(job_type, parameters, uow) - persisted_type = await uow.jobs.get_type(job_type) - if persisted_type is None: - raise UnknownJobTypeError(f"Unknown Job type: {job_type}") - effective_timeout = ( - persisted_type.default_timeout_seconds if timeout_seconds is None else timeout_seconds - ) - if effective_timeout <= 0: - raise ValueError("Job timeout_seconds must be positive") - - with operation("job.submit") as observation: - span = observation.span - carrier = submission_context() - job = JobModel( - type=job_type, - parameters=normalized, - timeout_seconds=effective_timeout, - submission_traceparent=carrier["submission_traceparent"], - submission_tracestate=carrier["submission_tracestate"], - ) - job = await uow.jobs.create(job) - span.set_attribute("inkcre.job.id", job.id or 0) - # The composing use case still owns commit; this span only proves the flush. - span.set_attribute("inkcre.job.write_phase", "flushed") - return job - - @classmethod - async def _prepare( - cls, - job: JobModel, - ) -> tuple[type[JobHandler], pydantic.BaseModel] | None: - handler = cls._handlers.get(job.type) - if handler is None: - return None - parameters = handler.validate_parameters(job.parameters) - return (handler, parameters) if await handler.can_handle(parameters) else None - - @classmethod - async def get(cls, job_id: JobID) -> JobModel | None: - async with job_uow() as uow: - return await uow.jobs.get(job_id) - - @classmethod - async def get_type(cls, type_: JobTypeID) -> JobTypeModel | None: - async with job_uow() as uow: - return await uow.jobs.get_type(type_) - - @classmethod - async def list_types( - cls, *, limit: int | None = None, cursor: str | None = None - ) -> tuple[list[JobTypeModel], str | None]: - async with job_uow() as uow: - return await uow.jobs.list_types(limit=limit, cursor=cursor) - - @classmethod - async def list_jobs( - cls, - *, - limit: int = 20, - cursor: JobID | None = None, - type_: JobTypeID | None = None, - status: JobStatus | None = None, - ) -> tuple[list[JobModel], JobID | None]: - async with job_uow() as uow: - return await uow.jobs.list_jobs( - limit=limit, cursor=cursor, type_=type_, status=status - ) - - @classmethod - async def notify_worker(cls) -> None: - """Best-effort post-commit hint; scan failure cannot undo accepted work.""" - try: - await cls.check() - except Exception: - LOGGER.exception("Job discovery hint failed; periodic discovery remains active") - - @classmethod - async def abort(cls, job_id: JobID) -> JobModel | None: - """Close pending work or request the executing Peer to stop running work.""" - async with job_uow() as uow: - return await uow.jobs.abort(job_id) - - @classmethod - async def check_abort_requests(cls) -> None: - """Read this Peer's active abort requests in one batch.""" - if not cls._active: - return - async with job_uow() as uow: - ids = await uow.jobs.abort_requests(tuple(cls._active)) - for job_id in ids: - if job_id is None: - continue - execution = cls._active.get(job_id) - if execution is not None: - execution.cancel() - - @classmethod - def start(cls) -> None: - cls._accepting = True - - @classmethod - async def shutdown(cls) -> None: - """Stop admission and drain handlers before their Extension resources close.""" - cls._accepting = False - executions = tuple(cls._active.values()) - for execution in executions: - execution.cancel() - await asyncio.gather(*(item.task for item in executions), return_exceptions=True) - - @classmethod - async def _claim(cls, job_id: JobID) -> JobModel | None: - if not cls._accepting: - return None - async with job_uow() as uow: - return await uow.jobs.claim(job_id) - - @classmethod - async def _close(cls, job: JobModel, status: JobStatus) -> bool: - if not status.terminal: - raise ValueError("Job may close only to a terminal status") - execution = cls._active.get(job.id) if job.id is not None else None - if execution is not None: - execution.closing = True - # Cleanup is independent of handler cancellation and cannot wait indefinitely. - async with asyncio.timeout(10): - async with job_uow() as uow: - closed = await uow.jobs.close(job, status) - if closed: - emit_event( - "job.closed", {"inkcre.job.id": job.id or 0, "inkcre.job.status": status.value} - ) - return closed - - @classmethod - async def run(cls, job_id: JobID) -> bool: - """Check local eligibility, atomically claim, then execute one Job.""" - if not cls._accepting: - return False - candidate = await cls.get(job_id) - if candidate is None or candidate.status != JobStatus.PENDING: - return False - try: - prepared = await cls._prepare(candidate) - except pydantic.ValidationError as error: - # A broken persisted command must be visible as failed, not remain pending - # forever. Unknown/unavailable handlers still leave work for another Peer. - LOGGER.exception("Persisted Job parameters are invalid", extra={"job_id": job_id}) - claimed = await cls._claim(job_id) - if claimed is None: - return False - claimed.state = {**claimed.state, "error": str(error)} - await cls._close(claimed, JobStatus.FAILED) - return True - if prepared is None: - return False - - claimed = await cls._claim(job_id) - if claimed is None: - return False - handler, parameters = prepared - task = asyncio.current_task() - assert task is not None - cls._active[job_id] = _Execution(task) - with operation( - "job.execute", - attributes={"inkcre.job.id": job_id}, - context=Context(), - links=submission_links(claimed.submission_traceparent, claimed.submission_tracestate), - ) as observation: - span = observation.span - emit_event("job.started", {"inkcre.job.id": job_id}) - try: - async with asyncio.timeout(claimed.timeout_seconds): - await handler.handle(claimed, parameters) - except asyncio.CancelledError: - LOGGER.info("Job execution aborted", extra={"job_id": job_id}) - observation.outcome = "cancelled" - span.set_attribute("inkcre.job.status", "aborted") - await cls._close(claimed, JobStatus.ABORTED) - except TimeoutError: - LOGGER.warning("Job execution timed out", extra={"job_id": job_id}) - observation.outcome = "error" - span.set_attribute("inkcre.job.status", "timed_out") - await cls._close(claimed, JobStatus.TIMED_OUT) - except Exception as error: - LOGGER.exception("Job execution failed", extra={"job_id": job_id}) - observation.outcome = "error" - span.set_attribute("error.type", type(error).__name__) - span.set_attribute("inkcre.job.status", "failed") - claimed.state = {**claimed.state, "error": str(error)} - await cls._close(claimed, JobStatus.FAILED) - else: - span.set_attribute("inkcre.job.status", "finished") - await cls._close(claimed, JobStatus.FINISHED) - finally: - cls._active.pop(job_id, None) - return True - - @classmethod - async def expire_overdue(cls) -> int: - """Use database time to close abandoned overdue running Jobs.""" - async with job_uow() as uow: - return await uow.jobs.expire_overdue() - - @classmethod - async def check(cls) -> None: - """Schedule locally eligible pending Jobs and converge running timeouts.""" - if not cls._accepting: - return - async with job_uow() as uow: - pending = await uow.jobs.pending() - - for job in pending: - if job.id is None or job.type not in cls._handlers: - continue - scheduler.add_job( - func=with_trace_id(f"job.{job.id}", cls.run), - args=[job.id], - id=f"job.{job.id}", - replace_existing=True, - misfire_grace_time=None, - ) - await cls.expire_overdue() +if TYPE_CHECKING: + from inkcre_core.business.job import * diff --git a/app/business/lexical_retrieval/__init__.py b/app/business/lexical_retrieval/__init__.py index 56144cdf..177a94d6 100644 --- a/app/business/lexical_retrieval/__init__.py +++ b/app/business/lexical_retrieval/__init__.py @@ -1,25 +1,3 @@ -"""Exact Block-local lexical retrieval capability.""" +"""Legacy import; use inkcre_core for new consumers.""" -from .main import ( - LEXICAL_RETRIEVAL_CAPABILITY, - LexicalRetrievalDelegationError, - LexicalRetrievalError, - LexicalRetrievalManager, -) -from .job import ( - LEXICAL_MAINTAIN_JOB_TYPE, - LEXICAL_REBUILD_JOB_TYPE, - LexicalMaintainJobHandler, - LexicalRebuildJobHandler, -) - -__all__ = [ - "LEXICAL_RETRIEVAL_CAPABILITY", - "LexicalRetrievalDelegationError", - "LexicalRetrievalError", - "LexicalRetrievalManager", - "LEXICAL_MAINTAIN_JOB_TYPE", - "LEXICAL_REBUILD_JOB_TYPE", - "LexicalMaintainJobHandler", - "LexicalRebuildJobHandler", -] +from inkcre_core.business.lexical_retrieval import * diff --git a/app/business/lexical_retrieval/job.py b/app/business/lexical_retrieval/job.py index a724f4db..b00d9b89 100644 --- a/app/business/lexical_retrieval/job.py +++ b/app/business/lexical_retrieval/job.py @@ -1,56 +1,10 @@ -"""Exact lexical projection maintenance commands hosted by Jobs.""" +"""Legacy import; use inkcre_core for new consumers.""" -from app.business.info_base.resolver import ResolverManager -from app.business.job import JobHandler -from app.schemas.job import JobModel -from app.schemas.lexical_retrieval import LexicalMaintenanceJobParameters +from typing import TYPE_CHECKING +from importlib import import_module +import sys -from .main import LexicalRetrievalManager +sys.modules[__name__] = import_module("inkcre_core.business.lexical_retrieval.job") - -LEXICAL_MAINTAIN_JOB_TYPE = "core.feature_retrieval.lexical.maintain.v1" -LEXICAL_REBUILD_JOB_TYPE = "core.feature_retrieval.lexical.rebuild.v1" - - -class LexicalMaintainJobHandler( - JobHandler[LexicalMaintenanceJobParameters], - job_type=LEXICAL_MAINTAIN_JOB_TYPE, - description="Maintain missing or stale Block lexical projection records.", - parameters_model=LexicalMaintenanceJobParameters, - default_timeout_seconds=900, -): - @classmethod - async def can_handle(cls, parameters: LexicalMaintenanceJobParameters) -> bool: - del parameters - return bool(ResolverManager.RESOLVER_CLS) - - @classmethod - async def handle( - cls, - job: JobModel, - parameters: LexicalMaintenanceJobParameters, - ) -> None: - report = await LexicalRetrievalManager.maintain(parameters.options) - job.state = report.model_dump(mode="json") - - -class LexicalRebuildJobHandler( - JobHandler[LexicalMaintenanceJobParameters], - job_type=LEXICAL_REBUILD_JOB_TYPE, - description="Rebuild Block lexical records present before invocation.", - parameters_model=LexicalMaintenanceJobParameters, - default_timeout_seconds=1800, -): - @classmethod - async def can_handle(cls, parameters: LexicalMaintenanceJobParameters) -> bool: - del parameters - return bool(ResolverManager.RESOLVER_CLS) - - @classmethod - async def handle( - cls, - job: JobModel, - parameters: LexicalMaintenanceJobParameters, - ) -> None: - report = await LexicalRetrievalManager.rebuild(parameters.options) - job.state = report.model_dump(mode="json") +if TYPE_CHECKING: + from inkcre_core.business.lexical_retrieval.job import * diff --git a/app/business/lexical_retrieval/main.py b/app/business/lexical_retrieval/main.py index a0bc3fac..a434a12f 100644 --- a/app/business/lexical_retrieval/main.py +++ b/app/business/lexical_retrieval/main.py @@ -1,261 +1,10 @@ -"""Block-local lexical projection maintenance and exact ranked retrieval.""" +"""Legacy import; use inkcre_core for new consumers.""" -from __future__ import annotations +from typing import TYPE_CHECKING +from importlib import import_module +import sys -from dataclasses import dataclass -import datetime -import logging -import typing +sys.modules[__name__] = import_module("inkcre_core.business.lexical_retrieval.main") -import pydantic - -from app.business.info_base.resolver import ( - ResolverManager, - UnknownResolverError, - UnsupportedResolverCapability, -) -from app.business.peer import PeerManager -from app.persistence.lexical_retrieval.uow import lexical_uow -from app.schemas.info_base.block import BlockModel -from app.schemas.lexical_retrieval import ( - LexicalEvidence, - LexicalMaintenanceDiagnostic, - LexicalMaintenanceOptions, - LexicalMaintenanceReport, - LexicalRetrievalMatch, - LexicalRetrievalRequest, - LexicalRetrievalResult, -) -from app.schemas.peer import PeerProtocolRequest, PeerProtocolResponse, PeerRef - - -logger = logging.getLogger(__name__) - -LEXICAL_RETRIEVAL_CAPABILITY = "core.feature_retrieval.lexical.v1" -_EXCERPT_LIMIT = 280 - - -class LexicalRetrievalError(RuntimeError): - pass - - -class LexicalRetrievalDelegationError(LexicalRetrievalError): - pass - - -class _ProjectionUnavailable(Exception): - pass - - -@dataclass(frozen=True) -class _Projection: - block: int - label: str - text: str | None - - -@dataclass -class _ReportBuilder: - diagnostic_limit: int - indexed: int = 0 - unavailable: int = 0 - failed: int = 0 - diagnostics: list[LexicalMaintenanceDiagnostic] | None = None - - def __post_init__(self) -> None: - self.diagnostics = [] - - def record( - self, - block: int, - outcome: typing.Literal["unavailable", "failed"], - reason: str, - ) -> None: - if outcome == "unavailable": - self.unavailable += 1 - else: - self.failed += 1 - diagnostics = typing.cast(list[LexicalMaintenanceDiagnostic], self.diagnostics) - if len(diagnostics) < self.diagnostic_limit: - diagnostics.append( - LexicalMaintenanceDiagnostic(block=block, outcome=outcome, reason=reason) - ) - - def build(self) -> LexicalMaintenanceReport: - return LexicalMaintenanceReport( - indexed=self.indexed, - unavailable=self.unavailable, - failed=self.failed, - diagnostics=tuple(self.diagnostics or ()), - ) - - -class LexicalRetrievalManager: - """Sole owner of lexical records, maintenance, ranking, and delegation.""" - - @classmethod - async def retrieve( - cls, - query: str, - limit: int = 20, - *, - route_to_peer: PeerRef | None = None, - ) -> LexicalRetrievalResult: - request = LexicalRetrievalRequest(query=query, limit=limit) - if route_to_peer is None or route_to_peer == PeerManager.get_current_peer_ref(): - return await cls.retrieve_local(request.query, request.limit) - - payload = PeerProtocolRequest( - body=typing.cast(typing.Any, request.model_dump(mode="json")) - ) - result = await PeerManager.delegate( - LEXICAL_RETRIEVAL_CAPABILITY, - typing.cast(typing.Any, payload.model_dump(mode="json", exclude_unset=True)), - route_to_peer=route_to_peer, - ) - try: - response = PeerProtocolResponse.model_validate(result) - if response.status != 200 or "body" not in response.model_fields_set: - raise LexicalRetrievalDelegationError( - f"Lexical retrieval Peer returned HTTP {response.status}" - ) - return LexicalRetrievalResult.model_validate(response.body) - except pydantic.ValidationError as error: - raise LexicalRetrievalDelegationError( - "Lexical retrieval Peer returned an invalid response" - ) from error - - @classmethod - async def retrieve_local(cls, query: str, limit: int = 20) -> LexicalRetrievalResult: - request = LexicalRetrievalRequest(query=query, limit=limit) - normalized = request.query.strip() - async with lexical_uow() as repository: - rows = await repository.search(normalized, request.limit) - - return LexicalRetrievalResult( - matches=tuple( - LexicalRetrievalMatch( - block=block, - label=record.label, - excerpt=cls._excerpt(record.label, record.text, normalized), - evidence=typing.cast(LexicalEvidence, evidence_value), - rank=float(rank_value), - ) - for block, record, evidence_value, rank_value in rows - ) - ) - - @classmethod - async def maintain( - cls, - options: LexicalMaintenanceOptions | None = None, - ) -> LexicalMaintenanceReport: - return await cls._maintain( - options or LexicalMaintenanceOptions(), - rebuild_cutoff=None, - ) - - @classmethod - async def rebuild( - cls, - options: LexicalMaintenanceOptions | None = None, - ) -> LexicalMaintenanceReport: - async with lexical_uow() as repository: - cutoff = await repository.database_now() - return await cls._maintain( - options or LexicalMaintenanceOptions(), - rebuild_cutoff=cutoff, - ) - - @classmethod - async def _maintain( - cls, - options: LexicalMaintenanceOptions, - *, - rebuild_cutoff: datetime.datetime | None, - ) -> LexicalMaintenanceReport: - report = _ReportBuilder(options.diagnostic_limit) - cursor = 0 - processed = 0 - while processed < options.max_records: - async with lexical_uow() as repository: - blocks, next_cursor, exhausted = await repository.candidate_page( - cursor, - min(options.scan_page_size, options.max_records - processed), - rebuild_cutoff, - ) - cursor = next_cursor - projections: list[_Projection] = [] - for block in blocks: - block_id = typing.cast(int, block.id) - processed += 1 - try: - projections.append(await cls._project(block)) - except _ProjectionUnavailable as error: - report.record(block_id, "unavailable", str(error)) - except Exception as error: - logger.exception("Lexical projection failed", extra={"block": block_id}) - report.record(block_id, "failed", type(error).__name__) - if projections: - try: - async with lexical_uow() as repository: - await repository.upsert( - [ - { - "block": projection.block, - "label": projection.label, - "text": projection.text, - } - for projection in projections - ] - ) - except Exception as error: - logger.exception( - "Lexical projection batch upsert failed", - extra={"batch_size": len(projections)}, - ) - for projection in projections: - report.record(projection.block, "failed", type(error).__name__) - else: - report.indexed += len(projections) - if exhausted: - break - return report.build() - - @classmethod - async def _project(cls, block: BlockModel) -> _Projection: - block_id = block.id - if block_id is None: - raise RuntimeError("Persisted lexical candidate is missing its ID") - try: - resolver = ResolverManager.get(block) - label = await resolver.get_label() - text = await resolver.get_text(context="lexical", materialize_missing=True) - except UnknownResolverError as error: - raise _ProjectionUnavailable("unknown_resolver") from error - except UnsupportedResolverCapability as error: - raise _ProjectionUnavailable("unsupported_lexical_text") from error - normalized_label = label.strip() - normalized_text = text.strip() if text is not None and text.strip() else None - if not normalized_label: - raise _ProjectionUnavailable("empty_label") - return _Projection(block_id, normalized_label, normalized_text) - - @staticmethod - def _excerpt(label: str, text: str | None, query: str) -> str: - source = text or label - folded_source = source.casefold() - needle = query.casefold() - position = folded_source.find(needle) - if position < 0: - positions = [ - candidate - for term in query.split() - if (candidate := folded_source.find(term.casefold())) >= 0 - ] - position = min(positions, default=0) - start = max(0, position - _EXCERPT_LIMIT // 3) - end = min(len(source), start + _EXCERPT_LIMIT) - prefix = "…" if start else "" - suffix = "…" if end < len(source) else "" - return f"{prefix}{source[start:end]}{suffix}" +if TYPE_CHECKING: + from inkcre_core.business.lexical_retrieval.main import * diff --git a/app/business/organization/__init__.py b/app/business/organization/__init__.py index 2d7c161c..f37c2a85 100644 --- a/app/business/organization/__init__.py +++ b/app/business/organization/__init__.py @@ -1,92 +1,3 @@ -"""Exact, extensible information-organization behaviors.""" +"""Legacy import; use inkcre_core for new consumers.""" -from .bootstrap import register_core_organization_behaviors -from .contracts import ( - OrganizationAgentNotFoundError, - OrganizationBlockNotFoundError, - OrganizationBudgetExceededError, - OrganizationDelegationError, - OrganizationError, - OrganizationExecutionError, - OrganizationNotConfiguredError, -) -from .duplicate_assertion import ( - DUPLICATES_ASSERTION_RELATION, - DuplicateAssertionBehaviorResolver, -) -from .evidence_stance import ( - CHALLENGES_RELATION, - SUPPORTS_RELATION, - EvidenceStanceBehaviorResolver, -) -from .referent_anchoring import ( - HAS_MENTION_RELATION, - REFERS_TO_RELATION, - ExistingReferentAnchoringBehaviorResolver, -) -from .refinement import REFINES_RELATION, RefinementBehaviorResolver -from .rumination import ( - RUMINATION_CAPABILITY, - RUMINATION_CONFIG_KEY, - RUMINATION_CONFIG_SCHEMA, - RuminationBehaviorResolver, -) -from .supersession import ( - EDITED_RELATION, - SUPERSEDES_RELATION, - SupersessionBehaviorResolver, -) -from .synthesis import SYNTHESIS_RELATION, SynthesisBehaviorResolver -from .tools import ( - ANCHOR_EXISTING_REFERENT_TOOL, - CREATE_SYNTHESIS_TOOL, - DRAFT_GRAPH_TOOL, - GET_DRAFT_GRAPH_SCHEMA_TOOL, - RECORD_DUPLICATE_ASSERTION_TOOL, - RECORD_EVIDENCE_STANCE_TOOL, - RECORD_ORGANIZATION_CANDIDATE_TOOL, - RECORD_REFINEMENT_TOOL, - RECORD_SUPERSESSION_TOOL, - SUBMIT_GRAPH_TOOL, -) - - -__all__ = [ - "ANCHOR_EXISTING_REFERENT_TOOL", - "CHALLENGES_RELATION", - "CREATE_SYNTHESIS_TOOL", - "DRAFT_GRAPH_TOOL", - "DUPLICATES_ASSERTION_RELATION", - "DuplicateAssertionBehaviorResolver", - "EDITED_RELATION", - "EvidenceStanceBehaviorResolver", - "ExistingReferentAnchoringBehaviorResolver", - "GET_DRAFT_GRAPH_SCHEMA_TOOL", - "HAS_MENTION_RELATION", - "OrganizationAgentNotFoundError", - "OrganizationBlockNotFoundError", - "OrganizationBudgetExceededError", - "OrganizationDelegationError", - "OrganizationError", - "OrganizationExecutionError", - "OrganizationNotConfiguredError", - "RECORD_DUPLICATE_ASSERTION_TOOL", - "RECORD_EVIDENCE_STANCE_TOOL", - "RECORD_ORGANIZATION_CANDIDATE_TOOL", - "RECORD_REFINEMENT_TOOL", - "RECORD_SUPERSESSION_TOOL", - "REFERS_TO_RELATION", - "REFINES_RELATION", - "RUMINATION_CAPABILITY", - "RUMINATION_CONFIG_KEY", - "RUMINATION_CONFIG_SCHEMA", - "RuminationBehaviorResolver", - "SUBMIT_GRAPH_TOOL", - "SUPERSEDES_RELATION", - "SUPPORTS_RELATION", - "SYNTHESIS_RELATION", - "SupersessionBehaviorResolver", - "SynthesisBehaviorResolver", - "RefinementBehaviorResolver", - "register_core_organization_behaviors", -] +from inkcre_core.business.organization import * diff --git a/app/business/organization/_shared.py b/app/business/organization/_shared.py index 8d3d7313..ff6a1830 100644 --- a/app/business/organization/_shared.py +++ b/app/business/organization/_shared.py @@ -1,362 +1,10 @@ -"""Small shared mechanics for exact Organization behaviors.""" +"""Legacy import; use inkcre_core for new consumers.""" -from __future__ import annotations +from typing import TYPE_CHECKING +from importlib import import_module +import sys -from contextlib import contextmanager -import json -import logging -import typing +sys.modules[__name__] = import_module("inkcre_core.business.organization._shared") -import pydantic - -from app.business.agent import AgentManager, AgentNotFoundError, TurnTermination -from app.business.deployment_config import DeploymentConfigService -from app.business.info_base.resolver import ( - Resolver, - ResolverManager, - UnknownResolverError, - UnsupportedResolverCapability, -) -from app.persistence.info_base.uow import GraphUnitOfWork, graph_uow -from app.schemas.ai import TextContentPart, UserMessage -from app.schemas.info_base.block import BlockForm, BlockID, BlockModel, ResolverType -from app.schemas.info_base.relation import RelationID, RelationModel -from app.schemas.organization_behavior import CandidateWriteResult, RelationWriteResult - -from .contracts import ( - OrganizationAgentNotFoundError, - OrganizationBlockNotFoundError, - OrganizationBudgetExceededError, - OrganizationExecutionError, - OrganizationNotConfiguredError, -) - - -CANDIDATE_RELATION = "candidate for" -_CONTEXT_RELATION_LIMIT = 20 - - -def behavior_resolver_classes() -> tuple[type[Resolver], ...]: - classes: list[type[Resolver]] = [] - for resolver_cls in ResolverManager.RESOLVER_CLS.values(): - if ( - issubclass(resolver_cls, Resolver) - and isinstance(getattr(resolver_cls, "organization_description", None), str) - and callable(getattr(resolver_cls, "record_candidate", None)) - ): - classes.append(resolver_cls) - return tuple(sorted(classes, key=lambda resolver_cls: resolver_cls.__rsotype__)) - - -def get_behavior_resolver(resolver: ResolverType) -> type[Resolver] | None: - return next( - ( - resolver_cls - for resolver_cls in behavior_resolver_classes() - if resolver_cls.__rsotype__ == resolver - ), - None, - ) - - -async def get_or_create_descriptor( - behavior: type[Resolver], - uow: GraphUnitOfWork, -) -> BlockModel: - form = BlockForm(resolver=behavior.__rsotype__, content="") - existing = await ResolverManager.get(BlockModel.model_validate(form)).get_existing_async( - uow.blocks - ) - return existing if existing is not None else await uow.blocks.create(form) - - -async def record_candidate( - behavior: type[Resolver], - block_id: BlockID, -) -> CandidateWriteResult: - async with graph_uow() as uow: - if (await uow.blocks.get(block_id)) is None: - raise OrganizationBlockNotFoundError(f"Block {block_id} does not exist") - descriptor = await get_or_create_descriptor(behavior, uow) - descriptor_id = _block_id(descriptor) - if block_id == descriptor_id: - raise ValueError("An Organization behavior cannot be its own candidate") - relation, created = await fetchsert_relation( - block_id, - descriptor_id, - CANDIDATE_RELATION, - uow, - ) - return CandidateWriteResult( - descriptor_block_id=descriptor_id, - relation_id=_relation_id(relation), - created=created, - ) - - -async def candidate_seed_ids( - behavior: type[Resolver], - limit: int, -) -> tuple[BlockID, ...]: - if limit <= 0: - return () - async with graph_uow() as uow: - descriptor = await get_or_create_descriptor(behavior, uow) - descriptor_id = _block_id(descriptor) - return await uow.relations.random_incoming_sources( - descriptor_id, CANDIDATE_RELATION, limit - ) - - -async def recent_block_ids(limit: int) -> tuple[BlockID, ...]: - if limit <= 0: - return () - behavior_types = tuple( - resolver_cls.__rsotype__ for resolver_cls in behavior_resolver_classes() - ) - async with graph_uow() as uow: - return await uow.blocks.select_ids( - limit, exclude_resolvers=behavior_types, random_order=False - ) - - -async def random_block_ids(limit: int) -> tuple[BlockID, ...]: - if limit <= 0: - return () - behavior_types = tuple( - resolver_cls.__rsotype__ for resolver_cls in behavior_resolver_classes() - ) - async with graph_uow() as uow: - return await uow.blocks.select_ids( - limit, exclude_resolvers=behavior_types, random_order=True - ) - - -async def recent_relation_endpoint_ids( - limit: int, - *, - contents: typing.Collection[str] = (), -) -> tuple[BlockID, ...]: - if limit <= 0: - return () - async with graph_uow() as uow: - relations = await uow.relations.recent(limit, contents=contents) - return tuple( - dict.fromkeys( - endpoint for relation in relations for endpoint in (relation.from_, relation.to_) - ) - ) - - -def merge_seed_categories( - max_seeds: int, - *categories: typing.Collection[BlockID], -) -> tuple[BlockID, ...]: - """Reserve one position per non-empty category, then fill in priority order.""" - result: list[BlockID] = [] - normalized = [tuple(dict.fromkeys(category)) for category in categories] - for category in normalized: - if category and category[0] not in result: - result.append(category[0]) - if len(result) == max_seeds: - return tuple(result) - for category in normalized: - for block_id in category[1:]: - if block_id not in result: - result.append(block_id) - if len(result) == max_seeds: - return tuple(result) - return tuple(result) - - -async def configured_agent_available( - config_key: str, - config_type: type[pydantic.BaseModel], -) -> bool: - config = await DeploymentConfigService.get(config_key) - if config is None: - return False - if not isinstance(config, config_type): - raise TypeError(f"Organization config {config_key!r} returned the wrong model") - return await AgentManager.can_execute(typing.cast(typing.Any, config).agent, "text") - - -@contextmanager -def continue_after_seed_failure( - logger: logging.Logger, - behavior: ResolverType, - seed_block_id: BlockID, -) -> typing.Generator[None, None, None]: - """Recover only candidate-local failures in an automatic attempt.""" - try: - yield - except (OrganizationBlockNotFoundError, OrganizationBudgetExceededError) as error: - logger.warning( - "organization.seed.considered", - extra={ - "behavior": behavior, - "seed_block_ids": (seed_block_id,), - "outcome": "recoverable_failure", - "reason": ( - "seed_missing" - if isinstance(error, OrganizationBlockNotFoundError) - else "model_call_limit" - ), - }, - ) - - -async def run_configured_agent( - config_key: str, - config_type: type[pydantic.BaseModel], - message: UserMessage, -) -> None: - config = await DeploymentConfigService.get(config_key) - if config is None: - raise OrganizationNotConfiguredError( - f"Organization behavior {config_key!r} is not configured" - ) - if not isinstance(config, config_type): - raise TypeError(f"Organization config {config_key!r} returned the wrong model") - agent_id = typing.cast(typing.Any, config).agent - try: - thread = await AgentManager.run(agent_id, message) - except AgentNotFoundError as error: - raise OrganizationAgentNotFoundError( - f"Configured Organization Agent {agent_id} does not exist" - ) from error - turn = thread.current_turn - if turn is None: # pragma: no cover - AgentManager.run invariant - raise OrganizationExecutionError("Organization Agent did not start a Turn") - outcome = await turn - if outcome == TurnTermination.MAX_MODEL_CALLS: - raise OrganizationBudgetExceededError( - "Organization Agent exceeded its per-Turn model-call budget" - ) - - -async def build_seed_message( - request: str, - judgment_contract: tuple[str, ...], - seed_id: BlockID, -) -> UserMessage | None: - """Resolve one bounded seed neighborhood without holding a DB transaction.""" - async with graph_uow() as uow: - block = await uow.blocks.get(seed_id) - if block is None: - raise OrganizationBlockNotFoundError(f"Block {seed_id} does not exist") - relations = tuple( - sorted( - (await uow.relations.get(seed_id)), - key=lambda relation: relation.id or 0, - )[-_CONTEXT_RELATION_LIMIT:] - ) - neighbor_ids = { - relation.to_ if relation.from_ == seed_id else relation.from_ - for relation in relations - } - neighbors = { - neighbor.id: neighbor - for neighbor in (await uow.blocks.get_many(neighbor_ids)) - if neighbor.id is not None - } - - try: - resolver = ResolverManager.get(block) - text = await resolver.get_text(materialize_missing=False) - label = await resolver.get_label() - except (UnknownResolverError, UnsupportedResolverCapability): - return None - if text is None or not text.strip(): - return None - - relation_context: list[dict[str, typing.Any]] = [] - for relation in relations: - other_id = relation.to_ if relation.from_ == seed_id else relation.from_ - other = neighbors.get(other_id) - other_label = None - if other is not None: - try: - other_label = await ResolverManager.get(other).get_label() - except (UnknownResolverError, UnsupportedResolverCapability): - pass - relation_context.append( - { - "id": relation.id, - "direction": "outgoing" if relation.from_ == seed_id else "incoming", - "content": relation.content, - "other_block": { - "id": other_id, - "resolver": other.resolver if other is not None else None, - "label": other_label, - }, - } - ) - context = { - "request": request, - "judgment_contract": judgment_contract, - "seed_block": { - "id": seed_id, - "resolver": block.resolver, - "label": label, - "text": text, - }, - "direct_relations": relation_context, - "exploration": ( - "The seed and its direct relations are only a starting point. Use the declared " - "retrieval, Resolver, and graph tools when more evidence is needed. Persist only " - "through the exact behavior tool, or cautiously mark a different behavior candidate." - ), - } - return UserMessage( - content=( - TextContentPart( - text=json.dumps( - context, - ensure_ascii=False, - sort_keys=True, - separators=(",", ":"), - ) - ), - ) - ) - - -async def fetchsert_relation( - from_: BlockID, - to_: BlockID, - content: str, - uow: GraphUnitOfWork, -) -> tuple[RelationModel, bool]: - proposed = RelationModel(from_=from_, to_=to_, content=content) - relation = await uow.relations.fetchsert(proposed) - return relation, relation is proposed - - -def relation_result(relation: RelationModel, created: bool) -> RelationWriteResult: - return RelationWriteResult(relation_id=_relation_id(relation), created=created) - - -async def require_distinct_blocks( - left: BlockID, - right: BlockID, - uow: GraphUnitOfWork, -) -> None: - if left == right: - raise ValueError("Organization relation endpoints must be different") - found = {block.id for block in (await uow.blocks.get_many((left, right)))} - missing = tuple(block_id for block_id in (left, right) if block_id not in found) - if missing: - raise OrganizationBlockNotFoundError(f"Blocks do not exist: {missing!r}") - - -def _block_id(block: BlockModel) -> BlockID: - if block.id is None: # pragma: no cover - persisted Block invariant - raise RuntimeError("Persisted Block has no ID") - return block.id - - -def _relation_id(relation: RelationModel) -> RelationID: - if relation.id is None: # pragma: no cover - persisted Relation invariant - raise RuntimeError("Persisted Relation has no ID") - return relation.id +if TYPE_CHECKING: + from inkcre_core.business.organization._shared import * diff --git a/app/business/organization/bootstrap.py b/app/business/organization/bootstrap.py index e235b03e..0de2ff4d 100644 --- a/app/business/organization/bootstrap.py +++ b/app/business/organization/bootstrap.py @@ -1,24 +1,10 @@ -"""Explicit loading of core Organization Behavior Resolvers.""" +"""Legacy import; use inkcre_core for new consumers.""" -from app.business.info_base.resolver import ResolverManager +from typing import TYPE_CHECKING +from importlib import import_module +import sys +sys.modules[__name__] = import_module("inkcre_core.business.organization.bootstrap") -def register_core_organization_behaviors() -> None: - from .duplicate_assertion import DuplicateAssertionBehaviorResolver - from .evidence_stance import EvidenceStanceBehaviorResolver - from .referent_anchoring import ExistingReferentAnchoringBehaviorResolver - from .refinement import RefinementBehaviorResolver - from .rumination import RuminationBehaviorResolver - from .supersession import SupersessionBehaviorResolver - from .synthesis import SynthesisBehaviorResolver - - for resolver_class in ( - RuminationBehaviorResolver, - SupersessionBehaviorResolver, - RefinementBehaviorResolver, - EvidenceStanceBehaviorResolver, - SynthesisBehaviorResolver, - ExistingReferentAnchoringBehaviorResolver, - DuplicateAssertionBehaviorResolver, - ): - ResolverManager.register_resolver(resolver_class) +if TYPE_CHECKING: + from inkcre_core.business.organization.bootstrap import * diff --git a/app/business/organization/contracts.py b/app/business/organization/contracts.py index a395a533..2c044ec0 100644 --- a/app/business/organization/contracts.py +++ b/app/business/organization/contracts.py @@ -1,29 +1,10 @@ -"""Stable failures at the Organization capability boundary.""" +"""Legacy import; use inkcre_core for new consumers.""" +from typing import TYPE_CHECKING +from importlib import import_module +import sys -class OrganizationError(RuntimeError): - """Base failure at the Organization capability boundary.""" +sys.modules[__name__] = import_module("inkcre_core.business.organization.contracts") - -class OrganizationBlockNotFoundError(OrganizationError): - pass - - -class OrganizationNotConfiguredError(OrganizationError): - pass - - -class OrganizationAgentNotFoundError(OrganizationError): - pass - - -class OrganizationExecutionError(OrganizationError): - pass - - -class OrganizationBudgetExceededError(OrganizationExecutionError): - """One Agent Turn ended at its configured model-call limit.""" - - -class OrganizationDelegationError(OrganizationError): - pass +if TYPE_CHECKING: + from inkcre_core.business.organization.contracts import * diff --git a/app/business/organization/duplicate_assertion.py b/app/business/organization/duplicate_assertion.py index 7f5aa4c0..fe76a6f7 100644 --- a/app/business/organization/duplicate_assertion.py +++ b/app/business/organization/duplicate_assertion.py @@ -1,160 +1,12 @@ -"""Provenance-aware duplicate assertion behavior.""" +"""Legacy import; use inkcre_core for new consumers.""" -from __future__ import annotations +from typing import TYPE_CHECKING +from importlib import import_module +import sys -import typing - - -from app.business.deployment_config import DeploymentConfigManager -from app.business.info_base.resolver import Resolver -from app.persistence.info_base.uow import graph_uow -from libs.obsrv.main import get_logger -from app.schemas.info_base.block import BlockID -from app.schemas.organization_behavior import ( - BehaviorAgentConfig, - CandidateWriteResult, - RelationWriteResult, -) - -from ._shared import ( - build_seed_message, - candidate_seed_ids, - configured_agent_available, - continue_after_seed_failure, - fetchsert_relation, - merge_seed_categories, - random_block_ids, - recent_block_ids, - record_candidate, - relation_result, - require_distinct_blocks, - run_configured_agent, -) - - -LOGGER = get_logger().getChild(__name__) - -DUPLICATES_ASSERTION_RELATION = "duplicates assertion" -DUPLICATE_ASSERTION_BEHAVIOR = "core.organization.behavior.duplicate-assertion.v1" -DUPLICATE_ASSERTION_CONFIG_KEY = "core.organization.duplicate_assertion" -DUPLICATE_ASSERTION_CONFIG_SCHEMA = "core.organization.duplicate_assertion.config.v1" - -DeploymentConfigManager.register_schema( - DUPLICATE_ASSERTION_CONFIG_SCHEMA, - BehaviorAgentConfig, - keys=(DUPLICATE_ASSERTION_CONFIG_KEY,), +sys.modules[__name__] = import_module( + "inkcre_core.business.organization.duplicate_assertion" ) - -class DuplicateAssertionBehaviorResolver( - Resolver[str, str], - rso_type=DUPLICATE_ASSERTION_BEHAVIOR, -): - organization_description = ( - "Relate whole-Block assertions copied from the same provenance occurrence." - ) - judgment_contract = ( - "Both Blocks completely and independently address the compared assertion.", - "Referent, predicate, polarity, force, units, and material qualifiers match.", - "Applicable scope, time, version, environment, and attribution are compatible.", - "Both assertions ultimately derive from the same observable provenance occurrence.", - "Neither Block adds independent evidence, reasoning, or authoritative decision.", - "No material asymmetric information gain is hidden by the relation.", - "The relation prevents evidence multiplication or restores useful provenance paths.", - ) - - async def get_text( - self, - *, - context: typing.Literal["default", "lexical"] = "default", - refresh: bool = False, - materialize_missing: bool = True, - ) -> str: - del context, refresh, materialize_missing - return self.organization_description - - async def get_label(self, *, refresh: bool = False) -> str: - del refresh - return "organization behavior: duplicate assertion" - - @classmethod - async def record_candidate( - cls, - block_id: BlockID, - ) -> CandidateWriteResult: - return await record_candidate(cls, block_id) - - @classmethod - async def record_duplicate_assertion( - cls, - left_block_id: BlockID, - right_block_id: BlockID, - ) -> RelationWriteResult: - async with graph_uow() as uow: - await require_distinct_blocks(left_block_id, right_block_id, uow) - from_, to_ = sorted((left_block_id, right_block_id)) - relation, created = await fetchsert_relation( - from_, - to_, - DUPLICATES_ASSERTION_RELATION, - uow, - ) - return relation_result(relation, created) - - @classmethod - async def can_run_automatic(cls) -> bool: - return await configured_agent_available( - DUPLICATE_ASSERTION_CONFIG_KEY, - BehaviorAgentConfig, - ) - - @classmethod - async def run_automatic(cls, max_seeds: int) -> None: - candidates = await candidate_seed_ids(cls, max_seeds) - recent = await recent_block_ids(max_seeds) - random = await random_block_ids(max_seeds) - seeds = merge_seed_categories(max_seeds, candidates, recent, random) - LOGGER.info( - "organization.seeds.selected", - extra={ - "behavior": cls.__rsotype__, - "max_seeds": max_seeds, - "candidate_count": len(candidates), - "recent_count": len(recent), - "random_count": len(random), - "seed_block_ids": seeds, - }, - ) - for seed in seeds: - with continue_after_seed_failure(LOGGER, cls.__rsotype__, seed): - message = await build_seed_message( - "Record only whole-Block duplicate assertions " - "from the same provenance occurrence.", - cls.judgment_contract, - seed, - ) - if message is None: - LOGGER.info( - "organization.seed.considered", - extra={ - "behavior": cls.__rsotype__, - "seed_block_ids": (seed,), - "outcome": "unresolved", - "reason": "text_unavailable", - }, - ) - continue - await run_configured_agent( - DUPLICATE_ASSERTION_CONFIG_KEY, - BehaviorAgentConfig, - message, - ) - LOGGER.info( - "organization.seed.considered", - extra={ - "behavior": cls.__rsotype__, - "seed_block_ids": (seed,), - "outcome": "considered", - "reason": "agent_completed", - }, - ) +if TYPE_CHECKING: + from inkcre_core.business.organization.duplicate_assertion import * diff --git a/app/business/organization/evidence_stance.py b/app/business/organization/evidence_stance.py index c92dd1ab..c60a0e1d 100644 --- a/app/business/organization/evidence_stance.py +++ b/app/business/organization/evidence_stance.py @@ -1,169 +1,10 @@ -"""Provenance-preserving evidence stance relation behavior.""" +"""Legacy import; use inkcre_core for new consumers.""" -from __future__ import annotations +from typing import TYPE_CHECKING +from importlib import import_module +import sys -import typing +sys.modules[__name__] = import_module("inkcre_core.business.organization.evidence_stance") - -from app.business.deployment_config import DeploymentConfigManager -from app.business.info_base.resolver import Resolver -from app.persistence.info_base.uow import graph_uow -from libs.obsrv.main import get_logger -from app.schemas.info_base.block import BlockID -from app.schemas.organization_behavior import ( - BehaviorAgentConfig, - CandidateWriteResult, - RelationWriteResult, -) - -from ._shared import ( - build_seed_message, - candidate_seed_ids, - configured_agent_available, - continue_after_seed_failure, - fetchsert_relation, - merge_seed_categories, - random_block_ids, - recent_block_ids, - recent_relation_endpoint_ids, - record_candidate, - relation_result, - require_distinct_blocks, - run_configured_agent, -) - - -LOGGER = get_logger().getChild(__name__) - -SUPPORTS_RELATION = "supports" -CHALLENGES_RELATION = "challenges" -EVIDENCE_STANCE_BEHAVIOR = "core.organization.behavior.evidence-stance.v1" -EVIDENCE_STANCE_CONFIG_KEY = "core.organization.evidence_stance" -EVIDENCE_STANCE_CONFIG_SCHEMA = "core.organization.evidence_stance.config.v1" - -DeploymentConfigManager.register_schema( - EVIDENCE_STANCE_CONFIG_SCHEMA, - BehaviorAgentConfig, - keys=(EVIDENCE_STANCE_CONFIG_KEY,), -) - - -class EvidenceStanceBehaviorResolver( - Resolver[str, str], - rso_type=EVIDENCE_STANCE_BEHAVIOR, -): - organization_description = ( - "Relate attributable evidence that supports or challenges an assertion." - ) - judgment_contract = ( - "Evidence and assertion are complete addressable information units.", - "The source is evidence and the target is an evaluable assertion.", - "Their proposition and applicable scope are comparable.", - "The evidence contributes reasons beyond establishing source fidelity; " - "source authority alone is insufficient.", - "Evidence provenance and speaker attribution remain recoverable.", - "The stance is unambiguously support or challenge for the whole assertion.", - "Duplicate, refinement, replacement, or topical proximity alone is insufficient.", - ) - - async def get_text( - self, - *, - context: typing.Literal["default", "lexical"] = "default", - refresh: bool = False, - materialize_missing: bool = True, - ) -> str: - del context, refresh, materialize_missing - return self.organization_description - - async def get_label(self, *, refresh: bool = False) -> str: - del refresh - return "organization behavior: evidence stance" - - @classmethod - async def record_candidate( - cls, - block_id: BlockID, - ) -> CandidateWriteResult: - return await record_candidate(cls, block_id) - - @classmethod - async def record_evidence_stance( - cls, - evidence_block_id: BlockID, - assertion_block_id: BlockID, - stance: typing.Literal["supports", "challenges"], - ) -> RelationWriteResult: - async with graph_uow() as uow: - await require_distinct_blocks(evidence_block_id, assertion_block_id, uow) - opposite = CHALLENGES_RELATION if stance == SUPPORTS_RELATION else SUPPORTS_RELATION - existing_opposite = await uow.relations.get_outgoing_many( - (evidence_block_id,), to_ids=(assertion_block_id,), content=opposite - ) - if existing_opposite: - raise ValueError("The evidence/assertion pair already has the opposite stance") - relation, created = await fetchsert_relation( - evidence_block_id, - assertion_block_id, - stance, - uow, - ) - return relation_result(relation, created) - - @classmethod - async def can_run_automatic(cls) -> bool: - return await configured_agent_available(EVIDENCE_STANCE_CONFIG_KEY, BehaviorAgentConfig) - - @classmethod - async def run_automatic(cls, max_seeds: int) -> None: - candidates = await candidate_seed_ids(cls, max_seeds) - strong = merge_seed_categories( - max_seeds, - await recent_relation_endpoint_ids(max_seeds), - await recent_block_ids(max_seeds), - ) - random = await random_block_ids(max_seeds) - seeds = merge_seed_categories(max_seeds, candidates, strong, random) - LOGGER.info( - "organization.seeds.selected", - extra={ - "behavior": cls.__rsotype__, - "max_seeds": max_seeds, - "candidate_count": len(candidates), - "strong_count": len(strong), - "random_count": len(random), - "seed_block_ids": seeds, - }, - ) - for seed in seeds: - with continue_after_seed_failure(LOGGER, cls.__rsotype__, seed): - message = await build_seed_message( - "Determine only attributable evidence support or challenge relations.", - cls.judgment_contract, - seed, - ) - if message is None: - LOGGER.info( - "organization.seed.considered", - extra={ - "behavior": cls.__rsotype__, - "seed_block_ids": (seed,), - "outcome": "unresolved", - "reason": "text_unavailable", - }, - ) - continue - await run_configured_agent( - EVIDENCE_STANCE_CONFIG_KEY, - BehaviorAgentConfig, - message, - ) - LOGGER.info( - "organization.seed.considered", - extra={ - "behavior": cls.__rsotype__, - "seed_block_ids": (seed,), - "outcome": "considered", - "reason": "agent_completed", - }, - ) +if TYPE_CHECKING: + from inkcre_core.business.organization.evidence_stance import * diff --git a/app/business/organization/jobs.py b/app/business/organization/jobs.py index b7e6b555..1c34ad60 100644 --- a/app/business/organization/jobs.py +++ b/app/business/organization/jobs.py @@ -1,197 +1,10 @@ -"""Independent automatic Organization routes and explicit rumination Jobs.""" +"""Legacy import; use inkcre_core for new consumers.""" -from app.business.job import JobHandler -from app.schemas.job import JobModel -from app.schemas.organization import RuminationRequest -from app.schemas.organization_behavior import AutomaticOrganizationJobParameters +from typing import TYPE_CHECKING +from importlib import import_module +import sys -from .duplicate_assertion import DuplicateAssertionBehaviorResolver -from .evidence_stance import EvidenceStanceBehaviorResolver -from .referent_anchoring import ExistingReferentAnchoringBehaviorResolver -from .refinement import RefinementBehaviorResolver -from .rumination import RuminationBehaviorResolver -from .supersession import SupersessionBehaviorResolver -from .synthesis import SynthesisBehaviorResolver +sys.modules[__name__] = import_module("inkcre_core.business.organization.jobs") - -RUMINATION_JOB_TYPE = "core.organization.rumination.automatic.v1" -EXPLICIT_RUMINATION_JOB_TYPE = "core.organization.rumination.explicit.v1" -SUPERSESSION_JOB_TYPE = "core.organization.supersession.automatic.v1" -REFINEMENT_JOB_TYPE = "core.organization.refinement.automatic.v1" -EVIDENCE_STANCE_JOB_TYPE = "core.organization.evidence-stance.automatic.v1" -SYNTHESIS_JOB_TYPE = "core.organization.synthesis.automatic.v1" -REFERENT_ANCHORING_JOB_TYPE = "core.organization.existing-referent-anchoring.automatic.v1" -DUPLICATE_ASSERTION_JOB_TYPE = "core.organization.duplicate-assertion.automatic.v1" - - -class ExplicitRuminationJobHandler( - JobHandler[RuminationRequest], - job_type=EXPLICIT_RUMINATION_JOB_TYPE, - description="Reconsider one explicitly selected Block through rumination.", - parameters_model=RuminationRequest, - default_timeout_seconds=1800, -): - @classmethod - async def can_handle(cls, parameters: RuminationRequest) -> bool: - del parameters - return await RuminationBehaviorResolver.can_run_automatic() - - @classmethod - async def handle(cls, job: JobModel, parameters: RuminationRequest) -> None: - del job - # A claimed Job executes here; it must not delegate another execution. - await RuminationBehaviorResolver.ruminate_local(parameters.block) - - -class RuminationJobHandler( - JobHandler[AutomaticOrganizationJobParameters], - job_type=RUMINATION_JOB_TYPE, - description="Automatically reconsider bounded information seeds through rumination.", - parameters_model=AutomaticOrganizationJobParameters, - default_timeout_seconds=1800, -): - @classmethod - async def can_handle(cls, parameters: AutomaticOrganizationJobParameters) -> bool: - del parameters - return await RuminationBehaviorResolver.can_run_automatic() - - @classmethod - async def handle( - cls, - job: JobModel, - parameters: AutomaticOrganizationJobParameters, - ) -> None: - del job - await RuminationBehaviorResolver.run_automatic(parameters.max_seeds) - - -class SupersessionJobHandler( - JobHandler[AutomaticOrganizationJobParameters], - job_type=SUPERSESSION_JOB_TYPE, - description="Automatically judge bounded scoped-supersession candidates.", - parameters_model=AutomaticOrganizationJobParameters, - default_timeout_seconds=1800, -): - @classmethod - async def can_handle(cls, parameters: AutomaticOrganizationJobParameters) -> bool: - del parameters - return await SupersessionBehaviorResolver.can_run_automatic() - - @classmethod - async def handle( - cls, - job: JobModel, - parameters: AutomaticOrganizationJobParameters, - ) -> None: - del job - await SupersessionBehaviorResolver.run_automatic(parameters.max_seeds) - - -class RefinementJobHandler( - JobHandler[AutomaticOrganizationJobParameters], - job_type=REFINEMENT_JOB_TYPE, - description="Automatically judge bounded non-dominating refinement candidates.", - parameters_model=AutomaticOrganizationJobParameters, - default_timeout_seconds=1800, -): - @classmethod - async def can_handle(cls, parameters: AutomaticOrganizationJobParameters) -> bool: - del parameters - return await RefinementBehaviorResolver.can_run_automatic() - - @classmethod - async def handle( - cls, - job: JobModel, - parameters: AutomaticOrganizationJobParameters, - ) -> None: - del job - await RefinementBehaviorResolver.run_automatic(parameters.max_seeds) - - -class EvidenceStanceJobHandler( - JobHandler[AutomaticOrganizationJobParameters], - job_type=EVIDENCE_STANCE_JOB_TYPE, - description="Automatically judge bounded attributable evidence stances.", - parameters_model=AutomaticOrganizationJobParameters, - default_timeout_seconds=1800, -): - @classmethod - async def can_handle(cls, parameters: AutomaticOrganizationJobParameters) -> bool: - del parameters - return await EvidenceStanceBehaviorResolver.can_run_automatic() - - @classmethod - async def handle( - cls, - job: JobModel, - parameters: AutomaticOrganizationJobParameters, - ) -> None: - del job - await EvidenceStanceBehaviorResolver.run_automatic(parameters.max_seeds) - - -class SynthesisJobHandler( - JobHandler[AutomaticOrganizationJobParameters], - job_type=SYNTHESIS_JOB_TYPE, - description="Automatically create bounded provenance-preserving syntheses.", - parameters_model=AutomaticOrganizationJobParameters, - default_timeout_seconds=1800, -): - @classmethod - async def can_handle(cls, parameters: AutomaticOrganizationJobParameters) -> bool: - del parameters - return await SynthesisBehaviorResolver.can_run_automatic() - - @classmethod - async def handle( - cls, - job: JobModel, - parameters: AutomaticOrganizationJobParameters, - ) -> None: - del job - await SynthesisBehaviorResolver.run_automatic(parameters.max_seeds) - - -class ExistingReferentAnchoringJobHandler( - JobHandler[AutomaticOrganizationJobParameters], - job_type=REFERENT_ANCHORING_JOB_TYPE, - description="Automatically anchor bounded mentions to existing referents.", - parameters_model=AutomaticOrganizationJobParameters, - default_timeout_seconds=1800, -): - @classmethod - async def can_handle(cls, parameters: AutomaticOrganizationJobParameters) -> bool: - del parameters - return await ExistingReferentAnchoringBehaviorResolver.can_run_automatic() - - @classmethod - async def handle( - cls, - job: JobModel, - parameters: AutomaticOrganizationJobParameters, - ) -> None: - del job - await ExistingReferentAnchoringBehaviorResolver.run_automatic(parameters.max_seeds) - - -class DuplicateAssertionJobHandler( - JobHandler[AutomaticOrganizationJobParameters], - job_type=DUPLICATE_ASSERTION_JOB_TYPE, - description="Automatically judge bounded provenance-aware duplicate assertions.", - parameters_model=AutomaticOrganizationJobParameters, - default_timeout_seconds=1800, -): - @classmethod - async def can_handle(cls, parameters: AutomaticOrganizationJobParameters) -> bool: - del parameters - return await DuplicateAssertionBehaviorResolver.can_run_automatic() - - @classmethod - async def handle( - cls, - job: JobModel, - parameters: AutomaticOrganizationJobParameters, - ) -> None: - del job - await DuplicateAssertionBehaviorResolver.run_automatic(parameters.max_seeds) +if TYPE_CHECKING: + from inkcre_core.business.organization.jobs import * diff --git a/app/business/organization/referent_anchoring.py b/app/business/organization/referent_anchoring.py index 3e20a73e..f13b3ec6 100644 --- a/app/business/organization/referent_anchoring.py +++ b/app/business/organization/referent_anchoring.py @@ -1,273 +1,12 @@ -"""Existing-referent anchoring through occurrence-local selected text.""" +"""Legacy import; use inkcre_core for new consumers.""" -from __future__ import annotations +from typing import TYPE_CHECKING +from importlib import import_module +import sys -import typing - - -from app.business.deployment_config import DeploymentConfigManager -from app.business.info_base.resolver import Resolver -from app.persistence.info_base.uow import GraphUnitOfWork, graph_uow -from libs.obsrv.main import get_logger -from app.schemas.info_base.block import BlockForm, BlockID -from app.schemas.info_base.relation import RelationModel -from app.schemas.organization_behavior import ( - BehaviorAgentConfig, - CandidateWriteResult, - ExistingReferentAnchorProposal, - ExistingReferentAnchorResult, -) - -from ._shared import ( - build_seed_message, - candidate_seed_ids, - configured_agent_available, - continue_after_seed_failure, - fetchsert_relation, - merge_seed_categories, - random_block_ids, - recent_block_ids, - record_candidate, - relation_result, - require_distinct_blocks, - run_configured_agent, +sys.modules[__name__] = import_module( + "inkcre_core.business.organization.referent_anchoring" ) - -LOGGER = get_logger().getChild(__name__) - -HAS_MENTION_RELATION = "has mention" -REFERS_TO_RELATION = "refers to" -REFERENT_ANCHORING_BEHAVIOR = "core.organization.behavior.existing-referent-anchoring.v1" -REFERENT_ANCHORING_CONFIG_KEY = "core.organization.existing_referent_anchoring" -REFERENT_ANCHORING_CONFIG_SCHEMA = "core.organization.existing_referent_anchoring.config.v1" - -DeploymentConfigManager.register_schema( - REFERENT_ANCHORING_CONFIG_SCHEMA, - BehaviorAgentConfig, - keys=(REFERENT_ANCHORING_CONFIG_KEY,), -) - - -class ExistingReferentAnchoringBehaviorResolver( - Resolver[str, str], - rso_type=REFERENT_ANCHORING_BEHAVIOR, -): - organization_description = ( - "Anchor one source-grounded referring fragment to existing " - "identity-bearing information." - ) - judgment_contract = ( - "The source expression meaningfully denotes a reusable referent.", - "Selected text identifies this mention without unrelated material.", - "The referent Block already exists and was not created as a temporary label.", - "The target contains enough identity to distinguish plausible alternatives.", - "Denotation remains continuous across name, time, environment, and scope.", - "Plausible competing referents have been considered and excluded.", - "The anchor improves cross-source or cross-time use rather than graph density.", - ) - - async def get_text( - self, - *, - context: typing.Literal["default", "lexical"] = "default", - refresh: bool = False, - materialize_missing: bool = True, - ) -> str: - del context, refresh, materialize_missing - return self.organization_description - - async def get_label(self, *, refresh: bool = False) -> str: - del refresh - return "organization behavior: existing referent anchoring" - - @classmethod - async def record_candidate( - cls, - block_id: BlockID, - ) -> CandidateWriteResult: - return await record_candidate(cls, block_id) - - @classmethod - async def anchor_existing_referent( - cls, - source_block_id: BlockID, - selected_text: str, - referent_block_id: BlockID, - ) -> ExistingReferentAnchorResult: - proposal = ExistingReferentAnchorProposal( - source_block_id=source_block_id, - selected_text=selected_text, - referent_block_id=referent_block_id, - ) - async with graph_uow() as uow: - await require_distinct_blocks( - proposal.source_block_id, proposal.referent_block_id, uow - ) - source = await uow.blocks.get(proposal.source_block_id) - if source is None: # pragma: no cover - require_distinct_blocks invariant - raise ValueError("Source Block does not exist") - - existing = await cls._existing_path( - proposal.source_block_id, - proposal.selected_text, - proposal.referent_block_id, - uow, - ) - if existing is not None: - fragment_id, has_mention, refers_to = existing - return ExistingReferentAnchorResult( - fragment_block_id=fragment_id, - fragment_created=False, - has_mention=( - relation_result(has_mention, False) if has_mention is not None else None - ), - refers_to=relation_result(refers_to, False), - ) - - source_is_fragment = ( - source.resolver == "core.text.v1" and source.content == proposal.selected_text - ) - if source_is_fragment: - fragment_id = proposal.source_block_id - fragment_created = False - has_mention_result = None - else: - fragment = await uow.blocks.create( - BlockForm(resolver="core.text.v1", content=proposal.selected_text) - ) - if fragment.id is None: # pragma: no cover - persisted Block invariant - raise RuntimeError("Persisted referring fragment has no ID") - fragment_id = fragment.id - fragment_created = True - has_mention, created = await fetchsert_relation( - proposal.source_block_id, - fragment_id, - HAS_MENTION_RELATION, - uow, - ) - has_mention_result = relation_result(has_mention, created) - - refers_to, created = await fetchsert_relation( - fragment_id, - proposal.referent_block_id, - REFERS_TO_RELATION, - uow, - ) - return ExistingReferentAnchorResult( - fragment_block_id=fragment_id, - fragment_created=fragment_created, - has_mention=has_mention_result, - refers_to=relation_result(refers_to, created), - ) - - @staticmethod - async def _existing_path( - source_block_id: BlockID, - selected_text: str, - referent_block_id: BlockID, - uow: GraphUnitOfWork, - ) -> tuple[BlockID, RelationModel | None, RelationModel] | None: - source = await uow.blocks.get(source_block_id) - if ( - source is not None - and source.resolver == "core.text.v1" - and source.content == selected_text - ): - refers_to = next( - ( - relation - for relation in ( - await uow.relations.get( - source_block_id, include_in=False, content=REFERS_TO_RELATION - ) - ) - if relation.to_ == referent_block_id - ), - None, - ) - if refers_to is not None: - return source_block_id, None, refers_to - - for has_mention in await uow.relations.get( - source_block_id, include_in=False, content=HAS_MENTION_RELATION - ): - fragment = await uow.blocks.get(has_mention.to_) - if ( - fragment is None - or fragment.resolver != "core.text.v1" - or fragment.content != selected_text - ): - continue - refers_to = next( - ( - relation - for relation in ( - await uow.relations.get( - has_mention.to_, include_in=False, content=REFERS_TO_RELATION - ) - ) - if relation.to_ == referent_block_id - ), - None, - ) - if refers_to is not None: - return has_mention.to_, has_mention, refers_to - return None - - @classmethod - async def can_run_automatic(cls) -> bool: - return await configured_agent_available( - REFERENT_ANCHORING_CONFIG_KEY, - BehaviorAgentConfig, - ) - - @classmethod - async def run_automatic(cls, max_seeds: int) -> None: - candidates = await candidate_seed_ids(cls, max_seeds) - recent = await recent_block_ids(max_seeds) - random = await random_block_ids(max_seeds) - seeds = merge_seed_categories(max_seeds, candidates, recent, random) - LOGGER.info( - "organization.seeds.selected", - extra={ - "behavior": cls.__rsotype__, - "max_seeds": max_seeds, - "candidate_count": len(candidates), - "recent_count": len(recent), - "random_count": len(random), - "seed_block_ids": seeds, - }, - ) - for seed in seeds: - with continue_after_seed_failure(LOGGER, cls.__rsotype__, seed): - message = await build_seed_message( - "Anchor only resolved source mentions to existing identity-bearing Blocks.", - cls.judgment_contract, - seed, - ) - if message is None: - LOGGER.info( - "organization.seed.considered", - extra={ - "behavior": cls.__rsotype__, - "seed_block_ids": (seed,), - "outcome": "unresolved", - "reason": "text_unavailable", - }, - ) - continue - await run_configured_agent( - REFERENT_ANCHORING_CONFIG_KEY, - BehaviorAgentConfig, - message, - ) - LOGGER.info( - "organization.seed.considered", - extra={ - "behavior": cls.__rsotype__, - "seed_block_ids": (seed,), - "outcome": "considered", - "reason": "agent_completed", - }, - ) +if TYPE_CHECKING: + from inkcre_core.business.organization.referent_anchoring import * diff --git a/app/business/organization/refinement.py b/app/business/organization/refinement.py index db380e3c..d7daa432 100644 --- a/app/business/organization/refinement.py +++ b/app/business/organization/refinement.py @@ -1,187 +1,10 @@ -"""Non-dominating refinement relation and automatic semantic judgment.""" +"""Legacy import; use inkcre_core for new consumers.""" -from __future__ import annotations +from typing import TYPE_CHECKING +from importlib import import_module +import sys -import typing +sys.modules[__name__] = import_module("inkcre_core.business.organization.refinement") - -from app.business.deployment_config import DeploymentConfigManager -from app.business.info_base.resolver import Resolver -from app.persistence.info_base.uow import GraphUnitOfWork, graph_uow -from libs.obsrv.main import get_logger -from app.schemas.info_base.block import BlockID -from app.schemas.organization_behavior import ( - BehaviorAgentConfig, - CandidateWriteResult, - RelationWriteResult, -) - -from ._shared import ( - build_seed_message, - candidate_seed_ids, - configured_agent_available, - continue_after_seed_failure, - fetchsert_relation, - merge_seed_categories, - random_block_ids, - recent_block_ids, - recent_relation_endpoint_ids, - record_candidate, - relation_result, - require_distinct_blocks, - run_configured_agent, -) -from .supersession import EDITED_RELATION - - -LOGGER = get_logger().getChild(__name__) - -REFINES_RELATION = "refines" -REFINEMENT_BEHAVIOR = "core.organization.behavior.refinement.v1" -REFINEMENT_CONFIG_KEY = "core.organization.refinement" -REFINEMENT_CONFIG_SCHEMA = "core.organization.refinement.config.v1" - -DeploymentConfigManager.register_schema( - REFINEMENT_CONFIG_SCHEMA, BehaviorAgentConfig, keys=(REFINEMENT_CONFIG_KEY,) -) - - -class RefinementBehaviorResolver( - Resolver[str, str], - rso_type=REFINEMENT_BEHAVIOR, -): - organization_description = ( - "Relate useful compatible detail that refines but does not replace information." - ) - judgment_contract = ( - "Both endpoints are complete addressable information units.", - "They continue the same referent and evolvable subject.", - "The refinement scope equals or is visibly contained by the predecessor scope.", - "Their information roles and attribution remain compatible.", - "The refinement adds reusable detail, constraints, explanation, or precision.", - "The predecessor remains independently safe as a coarser description.", - ) - - async def get_text( - self, - *, - context: typing.Literal["default", "lexical"] = "default", - refresh: bool = False, - materialize_missing: bool = True, - ) -> str: - del context, refresh, materialize_missing - return self.organization_description - - async def get_label(self, *, refresh: bool = False) -> str: - del refresh - return "organization behavior: refinement" - - @classmethod - async def record_candidate( - cls, - block_id: BlockID, - ) -> CandidateWriteResult: - return await record_candidate(cls, block_id) - - @classmethod - async def record_refinement( - cls, - refinement_block_id: BlockID, - predecessor_block_id: BlockID, - ) -> RelationWriteResult: - async with graph_uow() as uow: - await require_distinct_blocks(refinement_block_id, predecessor_block_id, uow) - if await cls._has_directed_path( - predecessor_block_id, - refinement_block_id, - uow=uow, - ): - raise ValueError( - f"Cannot refine: an existing refines path runs from predecessor " - f"{predecessor_block_id} to refinement {refinement_block_id}" - ) - relation, created = await fetchsert_relation( - refinement_block_id, - predecessor_block_id, - REFINES_RELATION, - uow, - ) - return relation_result(relation, created) - - @classmethod - async def _has_directed_path( - cls, - start: BlockID, - target: BlockID, - *, - uow: GraphUnitOfWork, - ) -> bool: - frontier = {start} - visited = {start} - while frontier: - rows = tuple( - relation.to_ - for relation in await uow.relations.get_outgoing_many( - frontier, content=REFINES_RELATION - ) - ) - next_frontier = set(rows) - visited - if target in next_frontier: - return True - visited.update(next_frontier) - frontier = next_frontier - return False - - @classmethod - async def can_run_automatic(cls) -> bool: - return await configured_agent_available(REFINEMENT_CONFIG_KEY, BehaviorAgentConfig) - - @classmethod - async def run_automatic(cls, max_seeds: int) -> None: - candidates = await candidate_seed_ids(cls, max_seeds) - strong = merge_seed_categories( - max_seeds, - await recent_relation_endpoint_ids(max_seeds, contents=(EDITED_RELATION,)), - await recent_block_ids(max_seeds), - ) - random = await random_block_ids(max_seeds) - seeds = merge_seed_categories(max_seeds, candidates, strong, random) - LOGGER.info( - "organization.seeds.selected", - extra={ - "behavior": cls.__rsotype__, - "max_seeds": max_seeds, - "candidate_count": len(candidates), - "strong_count": len(strong), - "random_count": len(random), - "seed_block_ids": seeds, - }, - ) - for seed in seeds: - with continue_after_seed_failure(LOGGER, cls.__rsotype__, seed): - message = await build_seed_message( - "Determine only useful non-dominating refinement relations.", - cls.judgment_contract, - seed, - ) - if message is None: - LOGGER.info( - "organization.seed.considered", - extra={ - "behavior": cls.__rsotype__, - "seed_block_ids": (seed,), - "outcome": "unresolved", - "reason": "text_unavailable", - }, - ) - continue - await run_configured_agent(REFINEMENT_CONFIG_KEY, BehaviorAgentConfig, message) - LOGGER.info( - "organization.seed.considered", - extra={ - "behavior": cls.__rsotype__, - "seed_block_ids": (seed,), - "outcome": "considered", - "reason": "agent_completed", - }, - ) +if TYPE_CHECKING: + from inkcre_core.business.organization.refinement import * diff --git a/app/business/organization/rumination.py b/app/business/organization/rumination.py index f9eeb397..841490f0 100644 --- a/app/business/organization/rumination.py +++ b/app/business/organization/rumination.py @@ -1,256 +1,10 @@ -"""Explicit and automatic rumination carried by an exact Behavior Resolver.""" +"""Legacy import; use inkcre_core for new consumers.""" -from __future__ import annotations +from typing import TYPE_CHECKING +from importlib import import_module +import sys -import json -import typing +sys.modules[__name__] = import_module("inkcre_core.business.organization.rumination") -import pydantic - -from app.business.deployment_config import DeploymentConfigManager -from app.business.info_base.resolver import ( - Resolver, - ResolverManager, - UnknownResolverError, - UnsupportedResolverCapability, -) -from app.business.peer import PeerManager -from app.persistence.info_base.uow import graph_uow -from libs.obsrv.main import get_logger -from app.schemas.ai import JSONValue, TextContentPart, UserMessage -from app.schemas.info_base.block import BlockID -from app.schemas.info_base.relation import RelationModel -from app.schemas.organization import RuminationConfig, RuminationRequest -from app.schemas.organization_behavior import CandidateWriteResult -from app.schemas.peer import PeerProtocolRequest, PeerProtocolResponse, PeerRef - -from ._shared import ( - candidate_seed_ids, - configured_agent_available, - continue_after_seed_failure, - merge_seed_categories, - random_block_ids, - recent_block_ids, - record_candidate, - run_configured_agent, -) -from .contracts import ( - OrganizationBlockNotFoundError, - OrganizationDelegationError, - OrganizationExecutionError, -) - - -LOGGER = get_logger().getChild(__name__) - -RUMINATION_CONFIG_KEY = "core.organization.rumination" -RUMINATION_CONFIG_SCHEMA = "core.organization.rumination.config.v1" -RUMINATION_CAPABILITY = "core.organization.rumination.v1" -RUMINATION_BEHAVIOR = "core.organization.behavior.rumination.v1" - -DeploymentConfigManager.register_schema( - RUMINATION_CONFIG_SCHEMA, RuminationConfig, keys=(RUMINATION_CONFIG_KEY,) -) - - -class RuminationBehaviorResolver( - Resolver[str, str], - rso_type=RUMINATION_BEHAVIOR, -): - organization_description = ( - "Open-ended reconsideration of one information Block that may add a useful graph." - ) - - async def get_text( - self, - *, - context: typing.Literal["default", "lexical"] = "default", - refresh: bool = False, - materialize_missing: bool = True, - ) -> str: - del context, refresh, materialize_missing - return self.organization_description - - async def get_label(self, *, refresh: bool = False) -> str: - del refresh - return "organization behavior: rumination" - - @classmethod - async def record_candidate( - cls, - block_id: BlockID, - ) -> CandidateWriteResult: - return await record_candidate(cls, block_id) - - @classmethod - async def can_run_automatic(cls) -> bool: - return await configured_agent_available(RUMINATION_CONFIG_KEY, RuminationConfig) - - @classmethod - async def run_automatic(cls, max_seeds: int) -> None: - candidates = await candidate_seed_ids(cls, max_seeds) - recent = await recent_block_ids(max_seeds) - random = await random_block_ids(max_seeds) - seeds = merge_seed_categories(max_seeds, candidates, recent, random) - LOGGER.info( - "organization.seeds.selected", - extra={ - "behavior": cls.__rsotype__, - "max_seeds": max_seeds, - "candidate_count": len(candidates), - "recent_count": len(recent), - "random_count": len(random), - "seed_block_ids": seeds, - }, - ) - for seed in seeds: - with continue_after_seed_failure(LOGGER, cls.__rsotype__, seed): - await cls.ruminate_local(seed) - LOGGER.info( - "organization.seed.considered", - extra={ - "behavior": cls.__rsotype__, - "seed_block_ids": (seed,), - "outcome": "considered", - "reason": "agent_completed", - }, - ) - - @classmethod - async def ruminate( - cls, - block_id: BlockID, - *, - route_to_peer: PeerRef | None = None, - ) -> None: - """Execute locally unless the caller explicitly selects another Peer.""" - request = RuminationRequest(block=block_id) - if route_to_peer is None or route_to_peer == PeerManager.get_current_peer_ref(): - await cls.ruminate_local(request.block) - return - - payload = PeerProtocolRequest( - body=typing.cast(JSONValue, request.model_dump(mode="json")) - ) - result = await PeerManager.delegate( - RUMINATION_CAPABILITY, - typing.cast(JSONValue, payload.model_dump(mode="json", exclude_unset=True)), - route_to_peer=route_to_peer, - ) - try: - response = PeerProtocolResponse.model_validate(result) - except pydantic.ValidationError as error: - raise OrganizationDelegationError( - "Rumination Peer returned an invalid response" - ) from error - if response.status != 204 or "body" in response.model_fields_set: - raise OrganizationDelegationError(f"Rumination Peer returned HTTP {response.status}") - - @classmethod - async def ruminate_local(cls, block_id: BlockID) -> None: - initial_message = await cls._build_initial_message(block_id) - if initial_message is None: - return - await run_configured_agent( - RUMINATION_CONFIG_KEY, - RuminationConfig, - initial_message, - ) - - @classmethod - async def _build_initial_message(cls, block_id: BlockID) -> UserMessage | None: - async with graph_uow() as uow: - block = await uow.blocks.get(block_id) - if block is None: - raise OrganizationBlockNotFoundError(f"Block {block_id} does not exist") - relations = tuple( - sorted( - (await uow.relations.get(block_id)), - key=lambda relation: relation.id or 0, - ) - ) - neighbor_ids = { - relation.to_ if relation.from_ == block_id else relation.from_ - for relation in relations - } - neighbors = { - neighbor.id: neighbor for neighbor in await uow.blocks.get_many(neighbor_ids) - } - - try: - focal_text = await ResolverManager.get(block).get_text() - except (UnknownResolverError, UnsupportedResolverCapability): - return None - except Exception as error: - raise OrganizationExecutionError( - "Rumination could not understand focal Block" - ) from error - if focal_text is None or not focal_text.strip(): - return None - - relation_context: list[dict[str, typing.Any]] = [] - for relation in relations: - neighbor_id, direction = cls._neighbor_and_direction(block_id, relation) - neighbor = neighbors.get(neighbor_id) - label: str | None = None - if neighbor is not None: - try: - label = await ResolverManager.get(neighbor).get_label() - except Exception: - LOGGER.debug( - "Could not project rumination neighbor label", - exc_info=True, - extra={"block": neighbor_id}, - ) - relation_context.append( - { - "id": relation.id, - "direction": direction, - "property": relation.content, - "other_block": { - "id": neighbor_id, - "resolver": neighbor.resolver if neighbor is not None else None, - "label": label, - }, - } - ) - - context = { - "request": "ruminate", - "focal_block": { - "id": block_id, - "resolver": block.resolver, - "text": focal_text, - }, - "direct_relations": relation_context, - "available_draft_resolvers": [ - { - "resolver": capability.resolver, - "description": capability.description, - } - for capability in ResolverManager.get_draft_capabilities() - ], - } - return UserMessage( - content=( - TextContentPart( - text=json.dumps( - context, - ensure_ascii=False, - sort_keys=True, - separators=(",", ":"), - ) - ), - ) - ) - - @staticmethod - def _neighbor_and_direction( - block_id: BlockID, - relation: RelationModel, - ) -> tuple[BlockID, typing.Literal["incoming", "outgoing", "self"]]: - if relation.from_ == block_id and relation.to_ == block_id: - return block_id, "self" - if relation.from_ == block_id: - return relation.to_, "outgoing" - return relation.from_, "incoming" +if TYPE_CHECKING: + from inkcre_core.business.organization.rumination import * diff --git a/app/business/organization/supersession.py b/app/business/organization/supersession.py index 14a7af87..bd87a186 100644 --- a/app/business/organization/supersession.py +++ b/app/business/organization/supersession.py @@ -1,305 +1,10 @@ -"""Scoped supersession relation and automatic semantic judgment.""" +"""Legacy import; use inkcre_core for new consumers.""" -from __future__ import annotations +from typing import TYPE_CHECKING +from importlib import import_module +import sys -from collections import deque -from graphlib import CycleError, TopologicalSorter -import typing +sys.modules[__name__] = import_module("inkcre_core.business.organization.supersession") - -from app.business.deployment_config import DeploymentConfigManager -from app.business.info_base.resolver import Resolver -from app.persistence.info_base.uow import GraphUnitOfWork, graph_uow -from libs.obsrv.main import get_logger -from app.schemas.graph_navigation_retrieval import GraphModel -from app.schemas.info_base.block import BlockID -from app.schemas.info_base.relation import RelationID, RelationModel -from app.schemas.organization_behavior import ( - BehaviorAgentConfig, - CandidateWriteResult, - RelationWriteResult, - SupersessionLineage, -) - -from ._shared import ( - build_seed_message, - candidate_seed_ids, - configured_agent_available, - continue_after_seed_failure, - fetchsert_relation, - merge_seed_categories, - random_block_ids, - recent_block_ids, - recent_relation_endpoint_ids, - record_candidate, - relation_result, - require_distinct_blocks, - run_configured_agent, -) - - -LOGGER = get_logger().getChild(__name__) - -SUPERSEDES_RELATION = "supersedes" -EDITED_RELATION = "edited" -SUPERSESSION_BEHAVIOR = "core.organization.behavior.supersession.v1" -SUPERSESSION_CONFIG_KEY = "core.organization.supersession" -SUPERSESSION_CONFIG_SCHEMA = "core.organization.supersession.config.v1" - -DeploymentConfigManager.register_schema( - SUPERSESSION_CONFIG_SCHEMA, - BehaviorAgentConfig, - keys=(SUPERSESSION_CONFIG_KEY,), -) - - -class SupersessionBehaviorResolver( - Resolver[str, str], - rso_type=SUPERSESSION_BEHAVIOR, -): - organization_description = ( - "Relate a semantic successor that fully replaces one predecessor in scope." - ) - judgment_contract = ( - "Both endpoints are complete addressable information units.", - "They continue the same referent and evolvable subject.", - "The successor covers the predecessor's complete applicable scope.", - "Semantic order, not collection time, identifies successor and predecessor.", - "The successor has authority for this subject and scope.", - "Continuing to use the predecessor as current would be wrong.", - ) - - async def get_text( - self, - *, - context: typing.Literal["default", "lexical"] = "default", - refresh: bool = False, - materialize_missing: bool = True, - ) -> str: - del context, refresh, materialize_missing - return self.organization_description - - async def get_label(self, *, refresh: bool = False) -> str: - del refresh - return "organization behavior: supersession" - - @classmethod - async def record_candidate( - cls, - block_id: BlockID, - ) -> CandidateWriteResult: - return await record_candidate(cls, block_id) - - @classmethod - async def record_supersession( - cls, - successor_block_id: BlockID, - predecessor_block_id: BlockID, - ) -> RelationWriteResult: - async with graph_uow() as uow: - await require_distinct_blocks(successor_block_id, predecessor_block_id, uow) - if await cls._has_directed_path( - predecessor_block_id, - successor_block_id, - uow=uow, - ): - raise ValueError( - f"Cannot supersede: an existing supersedes path runs from predecessor " - f"{predecessor_block_id} to successor {successor_block_id}" - ) - relation, created = await fetchsert_relation( - successor_block_id, - predecessor_block_id, - SUPERSEDES_RELATION, - uow, - ) - return relation_result(relation, created) - - @classmethod - async def _has_directed_path( - cls, - start: BlockID, - target: BlockID, - *, - uow: GraphUnitOfWork, - ) -> bool: - frontier = {start} - visited = {start} - while frontier: - rows = tuple( - relation.to_ - for relation in await uow.relations.get_outgoing_many( - frontier, content=SUPERSEDES_RELATION - ) - ) - next_frontier = set(rows) - visited - if target in next_frontier: - return True - visited.update(next_frontier) - frontier = next_frontier - return False - - async def read_lineage( - self, - focal_block_id: BlockID, - *, - max_explored_blocks: int = 1000, - max_explored_relations: int = 10000, - ) -> SupersessionLineage: - """Read supersedes history; incomplete or cyclic graphs have no current frontier.""" - if max_explored_blocks < 1 or max_explored_relations < 1: - raise ValueError("exploration bounds must be positive") - return await self._read_lineage( - focal_block_id, - max_explored_blocks=max_explored_blocks, - max_explored_relations=max_explored_relations, - ) - - async def _read_lineage( - self, - focal_block_id: BlockID, - *, - max_explored_blocks: int, - max_explored_relations: int, - ) -> SupersessionLineage: - async with graph_uow() as uow: - if (await uow.blocks.get(focal_block_id)) is None: - raise ValueError("Focal Block does not exist") - visited = {focal_block_id} - frontier = deque((focal_block_id,)) - relations: dict[RelationID, RelationModel] = {} - scanned = 0 - truncated = False - while frontier and not truncated: - current = frontier.popleft() - for endpoint in ("from", "to"): - cursor: RelationID | None = None - while not truncated: - remaining = max_explored_relations - scanned - if remaining == 0: - truncated = True - break - requested = min(200, remaining + 1) - page = await uow.relations.get_endpoint_page( - (current,), - endpoint=typing.cast(typing.Literal["from", "to"], endpoint), - contents=(SUPERSEDES_RELATION,), - cursor=cursor, - limit=requested, - ) - if len(page) > remaining: - page = page[:remaining] - truncated = True - scanned += len(page) - for relation in page: - if relation.id is None: - continue - relations[relation.id] = relation - neighbor = relation.to_ if relation.from_ == current else relation.from_ - if neighbor in visited: - continue - if len(visited) >= max_explored_blocks: - truncated = True - break - visited.add(neighbor) - frontier.append(neighbor) - if truncated or len(page) < requested: - break - cursor = typing.cast(RelationID, page[-1].id) - - cycle_detected = self._cycle_detected(visited, tuple(relations.values())) - incoming = {relation.to_ for relation in relations.values()} - current_block_ids = ( - () - if truncated or cycle_detected - else tuple(sorted(block_id for block_id in visited if block_id not in incoming)) - ) - blocks = await uow.blocks.get_many(visited) - existing = {block.id for block in blocks} - closed_relations = tuple( - relation - for relation in relations.values() - if relation.from_ in existing and relation.to_ in existing - ) - return SupersessionLineage( - graph=GraphModel(blocks=blocks, relations=closed_relations), - current_block_ids=current_block_ids, - truncated=truncated, - cycle_detected=cycle_detected, - ) - - @staticmethod - def _cycle_detected( - blocks: typing.Collection[BlockID], - relations: typing.Collection[RelationModel], - ) -> bool: - # The standard library uses an explicit stack, so long lineages do not - # consume Python call-stack depth. - sorter: TopologicalSorter[BlockID] = TopologicalSorter() - for block_id in blocks: - sorter.add(block_id) - for relation in relations: - sorter.add(relation.to_, relation.from_) - try: - sorter.prepare() - except CycleError: - return True - return False - - @classmethod - async def can_run_automatic(cls) -> bool: - return await configured_agent_available(SUPERSESSION_CONFIG_KEY, BehaviorAgentConfig) - - @classmethod - async def run_automatic(cls, max_seeds: int) -> None: - candidates = await candidate_seed_ids(cls, max_seeds) - strong = merge_seed_categories( - max_seeds, - await recent_relation_endpoint_ids(max_seeds, contents=(EDITED_RELATION,)), - await recent_block_ids(max_seeds), - ) - random = await random_block_ids(max_seeds) - seeds = merge_seed_categories(max_seeds, candidates, strong, random) - LOGGER.info( - "organization.seeds.selected", - extra={ - "behavior": cls.__rsotype__, - "max_seeds": max_seeds, - "candidate_count": len(candidates), - "strong_count": len(strong), - "random_count": len(random), - "seed_block_ids": seeds, - }, - ) - for seed in seeds: - with continue_after_seed_failure(LOGGER, cls.__rsotype__, seed): - message = await build_seed_message( - "Determine only well-supported scoped supersession relations.", - cls.judgment_contract, - seed, - ) - if message is None: - LOGGER.info( - "organization.seed.considered", - extra={ - "behavior": cls.__rsotype__, - "seed_block_ids": (seed,), - "outcome": "unresolved", - "reason": "text_unavailable", - }, - ) - continue - await run_configured_agent( - SUPERSESSION_CONFIG_KEY, - BehaviorAgentConfig, - message, - ) - LOGGER.info( - "organization.seed.considered", - extra={ - "behavior": cls.__rsotype__, - "seed_block_ids": (seed,), - "outcome": "considered", - "reason": "agent_completed", - }, - ) +if TYPE_CHECKING: + from inkcre_core.business.organization.supersession import * diff --git a/app/business/organization/synthesis.py b/app/business/organization/synthesis.py index 755a1282..3cc75932 100644 --- a/app/business/organization/synthesis.py +++ b/app/business/organization/synthesis.py @@ -1,236 +1,10 @@ -"""Provenance-preserving n-ary synthesis behavior.""" +"""Legacy import; use inkcre_core for new consumers.""" -from __future__ import annotations +from typing import TYPE_CHECKING +from importlib import import_module +import sys -import typing +sys.modules[__name__] = import_module("inkcre_core.business.organization.synthesis") - -from app.business.deployment_config import DeploymentConfigManager -from app.business.info_base.resolver import Resolver -from app.persistence.info_base.uow import GraphUnitOfWork, graph_uow -from libs.obsrv.main import get_logger -from app.schemas.info_base.block import BlockForm, BlockID, BlockModel -from app.schemas.organization_behavior import ( - BehaviorAgentConfig, - CandidateWriteResult, - SynthesisProposal, - SynthesisWriteResult, -) - -from ._shared import ( - build_seed_message, - candidate_seed_ids, - configured_agent_available, - continue_after_seed_failure, - fetchsert_relation, - merge_seed_categories, - random_block_ids, - recent_block_ids, - recent_relation_endpoint_ids, - record_candidate, - relation_result, - run_configured_agent, -) -from .supersession import EDITED_RELATION - - -LOGGER = get_logger().getChild(__name__) - -SYNTHESIS_RELATION = "synthesis" -SYNTHESIS_BEHAVIOR = "core.organization.behavior.synthesis.v1" -SYNTHESIS_CONFIG_KEY = "core.organization.synthesis" -SYNTHESIS_CONFIG_SCHEMA = "core.organization.synthesis.config.v1" - -DeploymentConfigManager.register_schema( - SYNTHESIS_CONFIG_SCHEMA, BehaviorAgentConfig, keys=(SYNTHESIS_CONFIG_KEY,) -) - - -class SynthesisBehaviorResolver( - Resolver[str, str], - rso_type=SYNTHESIS_BEHAVIOR, -): - organization_description = ( - "Create reusable multi-source information while preserving exact source basis." - ) - judgment_contract = ( - "The result is independently useful information, not a list of related sources.", - "Every source materially contributes content, attribution, or uncertainty.", - "Source disagreements, uncertainty, and speaker attribution remain visible.", - "Scopes are compatible or their differences are explicitly preserved.", - "Duplicate-connected copies do not multiply independent corroboration.", - "No existing synthesis already provides the same reusable distinction.", - "Observed past use and recurrence make future reuse plausible.", - ) - - async def get_text( - self, - *, - context: typing.Literal["default", "lexical"] = "default", - refresh: bool = False, - materialize_missing: bool = True, - ) -> str: - del context, refresh, materialize_missing - return self.organization_description - - async def get_label(self, *, refresh: bool = False) -> str: - del refresh - return "organization behavior: synthesis" - - @classmethod - async def record_candidate( - cls, - block_id: BlockID, - ) -> CandidateWriteResult: - return await record_candidate(cls, block_id) - - @classmethod - async def create_synthesis( - cls, - text: str, - source_block_ids: typing.Collection[BlockID], - previous_synthesis_block_id: BlockID | None = None, - ) -> SynthesisWriteResult: - proposal = SynthesisProposal( - text=text, - source_block_ids=tuple(source_block_ids), - previous_synthesis_block_id=previous_synthesis_block_id, - ) - async with graph_uow() as uow: - found = {block.id for block in (await uow.blocks.get_many(proposal.source_block_ids))} - missing = tuple(source for source in proposal.source_block_ids if source not in found) - if missing: - raise ValueError(f"Synthesis source Blocks do not exist: {missing!r}") - if proposal.previous_synthesis_block_id is not None: - previous = await uow.blocks.get(proposal.previous_synthesis_block_id) - if previous is None: - raise ValueError("Previous synthesis Block does not exist") - previous_basis = await cls._source_basis(previous, uow) - if len(set(previous_basis)) < 2: - raise ValueError("Previous synthesis has no valid multi-source basis") - - source_set = set(proposal.source_block_ids) - text_candidates = await uow.blocks.matching_content("core.text.v1", proposal.text) - synthesis = None - for candidate in text_candidates: - if await cls._source_basis(candidate, uow) == source_set: - synthesis = candidate - break - synthesis_created = synthesis is None - if synthesis is None: - synthesis = await uow.blocks.create( - BlockForm(resolver="core.text.v1", content=proposal.text) - ) - if synthesis.id is None: # pragma: no cover - persisted Block invariant - raise RuntimeError("Persisted synthesis Block has no ID") - - basis = [] - for source_block_id in sorted(source_set): - relation, created = await fetchsert_relation( - source_block_id, - synthesis.id, - SYNTHESIS_RELATION, - uow, - ) - basis.append(relation_result(relation, created)) - - edited = None - if ( - proposal.previous_synthesis_block_id is not None - and proposal.previous_synthesis_block_id != synthesis.id - ): - relation, created = await fetchsert_relation( - proposal.previous_synthesis_block_id, - synthesis.id, - EDITED_RELATION, - uow, - ) - edited = relation_result(relation, created) - return SynthesisWriteResult( - synthesis_block_id=synthesis.id, - synthesis_created=synthesis_created, - basis=tuple(basis), - edited=edited, - ) - - @staticmethod - async def _source_basis( - synthesis: BlockModel, - uow: GraphUnitOfWork, - ) -> set[BlockID]: - if synthesis.id is None: - return set() - return { - relation.from_ - for relation in await uow.relations.get( - synthesis.id, include_out=False, content=SYNTHESIS_RELATION - ) - } - - @classmethod - async def _change_signal_ids(cls, limit: int) -> tuple[BlockID, ...]: - endpoints = await recent_relation_endpoint_ids(limit) - if not endpoints: - return () - async with graph_uow() as uow: - affected = tuple( - relation.to_ - for relation in await uow.relations.get_outgoing_many( - endpoints, content=SYNTHESIS_RELATION - ) - ) - return tuple(dict.fromkeys((*endpoints, *affected))) - - @classmethod - async def can_run_automatic(cls) -> bool: - return await configured_agent_available(SYNTHESIS_CONFIG_KEY, BehaviorAgentConfig) - - @classmethod - async def run_automatic(cls, max_seeds: int) -> None: - candidates = await candidate_seed_ids(cls, max_seeds) - strong = merge_seed_categories( - max_seeds, - await cls._change_signal_ids(max_seeds), - await recent_block_ids(max_seeds), - ) - random = await random_block_ids(max_seeds) - seeds = merge_seed_categories(max_seeds, candidates, strong, random) - LOGGER.info( - "organization.seeds.selected", - extra={ - "behavior": cls.__rsotype__, - "max_seeds": max_seeds, - "candidate_count": len(candidates), - "strong_count": len(strong), - "random_count": len(random), - "seed_block_ids": seeds, - }, - ) - for seed in seeds: - with continue_after_seed_failure(LOGGER, cls.__rsotype__, seed): - message = await build_seed_message( - "Create only provenance-preserving, reusable multi-source synthesis.", - cls.judgment_contract, - seed, - ) - if message is None: - LOGGER.info( - "organization.seed.considered", - extra={ - "behavior": cls.__rsotype__, - "seed_block_ids": (seed,), - "outcome": "unresolved", - "reason": "text_unavailable", - }, - ) - continue - await run_configured_agent(SYNTHESIS_CONFIG_KEY, BehaviorAgentConfig, message) - LOGGER.info( - "organization.seed.considered", - extra={ - "behavior": cls.__rsotype__, - "seed_block_ids": (seed,), - "outcome": "considered", - "reason": "agent_completed", - }, - ) +if TYPE_CHECKING: + from inkcre_core.business.organization.synthesis import * diff --git a/app/business/organization/tools.py b/app/business/organization/tools.py index d2a1f245..a65c0b06 100644 --- a/app/business/organization/tools.py +++ b/app/business/organization/tools.py @@ -1,342 +1,10 @@ -"""Owner-coherent Agent read tools and exact Organization mutation tools.""" +"""Legacy import; use inkcre_core for new consumers.""" -from __future__ import annotations +from typing import TYPE_CHECKING +from importlib import import_module +import sys -import typing +sys.modules[__name__] = import_module("inkcre_core.business.organization.tools") -import pydantic - -from app.business.agent import AgentManager, ToolExecutionError -from app.business.info_base import InfoBaseManager -from app.business.info_base.commands import submit_graph as persist_submitted_graph -from app.business.info_base.resolver import ( - ResolverDraftCapability, - ResolverManager, -) -from app.schemas.ai import JSONValue -from app.schemas.organization import ( - DraftGraphInput, - GetDraftGraphSchemaInput, - SubmitGraphInput, -) -from app.schemas.organization_behavior import ( - DuplicateAssertionProposal, - EvidenceStanceProposal, - ExistingReferentAnchorProposal, - RecordOrganizationCandidateInput, - RefinementProposal, - SupersessionProposal, - SynthesisProposal, -) -from ._shared import behavior_resolver_classes, get_behavior_resolver -from .contracts import OrganizationError -from .duplicate_assertion import ( - DUPLICATES_ASSERTION_RELATION, - DuplicateAssertionBehaviorResolver, -) -from .evidence_stance import EvidenceStanceBehaviorResolver -from .referent_anchoring import ExistingReferentAnchoringBehaviorResolver -from .refinement import REFINES_RELATION, RefinementBehaviorResolver -from .supersession import SUPERSEDES_RELATION, SupersessionBehaviorResolver -from .synthesis import SynthesisBehaviorResolver - - -GET_DRAFT_GRAPH_SCHEMA_TOOL = "get_draft_graph_schema" -DRAFT_GRAPH_TOOL = "draft_graph" -SUBMIT_GRAPH_TOOL = "submit_graph" -RECORD_SUPERSESSION_TOOL = "record_supersession" -RECORD_REFINEMENT_TOOL = "record_refinement" -RECORD_EVIDENCE_STANCE_TOOL = "record_evidence_stance" -CREATE_SYNTHESIS_TOOL = "create_synthesis" -ANCHOR_EXISTING_REFERENT_TOOL = "anchor_existing_referent" -RECORD_DUPLICATE_ASSERTION_TOOL = "record_duplicate_assertion" -RECORD_ORGANIZATION_CANDIDATE_TOOL = "record_organization_candidate" - - -def _draft_capability_snapshot() -> dict[str, ResolverDraftCapability]: - return { - capability.resolver: capability - for capability in ResolverManager.get_draft_capabilities() - } - - -def _schema_discovery_input_model() -> type[pydantic.BaseModel]: - snapshot = _draft_capability_snapshot() - if not snapshot: # pragma: no cover - core.text.v1 is always registered - raise RuntimeError("No Resolver graph-drafting capability is registered") - - def add_exact_ids(schema: dict[str, typing.Any]) -> None: - schema["properties"]["resolver_types"]["items"] = { - "type": "string", - "enum": list(snapshot), - } - - class BoundGetDraftGraphSchemaInput(GetDraftGraphSchemaInput): - model_config = pydantic.ConfigDict( - extra="forbid", - frozen=True, - json_schema_extra=add_exact_ids, - ) - - @pydantic.field_validator("resolver_types") - @classmethod - def exact_resolvers(cls, resolvers: tuple[str, ...]) -> tuple[str, ...]: - unknown = tuple(resolver for resolver in resolvers if resolver not in snapshot) - if unknown: - raise ValueError(f"Unavailable draft Resolver IDs: {unknown!r}") - return resolvers - - return BoundGetDraftGraphSchemaInput - - -def _draft_graph_input_model() -> type[pydantic.BaseModel]: - snapshot = _draft_capability_snapshot() - if not snapshot: # pragma: no cover - core.text.v1 is always registered - raise RuntimeError("No Resolver graph-drafting capability is registered") - - def add_exact_ids(schema: dict[str, typing.Any]) -> None: - schema["properties"]["resolver_type"] = { - "type": "string", - "enum": list(snapshot), - } - - class BoundDraftGraphInput(DraftGraphInput): - model_config = pydantic.ConfigDict( - extra="forbid", - json_schema_extra=add_exact_ids, - ) - - @pydantic.field_validator("resolver_type") - @classmethod - def exact_resolver(cls, resolver: str) -> str: - if resolver not in snapshot: - raise ValueError(f"Unavailable draft Resolver ID: {resolver!r}") - return resolver - - @pydantic.model_validator(mode="after") - def validate_resolver_input(self) -> typing.Self: - capability = snapshot[self.resolver_type] - # Validate at the caller's actual path; an inner resolver_type error must - # not tell the Agent to remove its valid outer selector. - payload_model = pydantic.create_model( - "ResolverDraftPayload", input=(capability.input_model, ...) - ) - payload = payload_model.model_validate({"input": self.input}) - object.__setattr__(self, "_resolver_input", getattr(payload, "input")) - return self - - return BoundDraftGraphInput - - -@AgentManager.tool( - GET_DRAFT_GRAPH_SCHEMA_TOOL, - description="Describe graph-drafting inputs for selected Resolver types.", - input_model_factory=_schema_discovery_input_model, -) -async def get_draft_graph_schema(input: GetDraftGraphSchemaInput) -> JSONValue: - snapshot = _draft_capability_snapshot() - return { - "resolvers": [ - { - "resolver_type": resolver, - "description": snapshot[resolver].description, - "input_schema": typing.cast( - dict[str, JSONValue], - snapshot[resolver].input_model.model_json_schema(), - ), - } - for resolver in input.resolver_types - ] - } - - -@AgentManager.tool( - DRAFT_GRAPH_TOOL, - description="Draft one rooted GraphForm through an exact Resolver without persistence.", - input_model_factory=_draft_graph_input_model, -) -async def draft_graph(input: DraftGraphInput) -> JSONValue: - capability = ResolverManager.get_draft_capability(input.resolver_type) - resolver_input = typing.cast(pydantic.BaseModel, getattr(input, "_resolver_input")) - stars = capability.resolver_cls.create_graph(resolver_input) - graph = InfoBaseManager.normalize_graph(stars, input.local_block_id_start) - return typing.cast(JSONValue, graph.model_dump(mode="json")) - - -@AgentManager.tool( - SUBMIT_GRAPH_TOOL, - description="Persist one complete GraphForm and return local-to-persisted Block IDs.", -) -async def submit_graph(input: SubmitGraphInput) -> JSONValue: - result = await persist_submitted_graph(input.graph) - return typing.cast(JSONValue, result.model_dump(mode="json")) - - -def _candidate_input_model() -> type[pydantic.BaseModel]: - snapshot = {behavior.__rsotype__: behavior for behavior in behavior_resolver_classes()} - if not snapshot: - raise RuntimeError("No Organization Behavior Resolver is registered") - - def add_exact_behaviors(schema: dict[str, typing.Any]) -> None: - schema["properties"]["behavior"] = { - "oneOf": [ - { - "const": behavior_id, - "description": behavior.organization_description, - } - for behavior_id, behavior in snapshot.items() - ] - } - - class BoundRecordOrganizationCandidateInput(RecordOrganizationCandidateInput): - model_config = pydantic.ConfigDict( - extra="forbid", - frozen=True, - json_schema_extra=add_exact_behaviors, - ) - - @pydantic.field_validator("behavior") - @classmethod - def exact_behavior(cls, behavior: str) -> str: - if behavior not in snapshot: - raise ValueError( - f"Unavailable behavior {behavior!r}; available: {', '.join(snapshot)}" - ) - return behavior - - return BoundRecordOrganizationCandidateInput - - -async def _exact_result(operation: typing.Awaitable[pydantic.BaseModel]) -> JSONValue: - try: - result = await operation - except (ValueError, OrganizationError) as error: - raise ToolExecutionError( - {"error": type(error).__name__, "message": str(error)} - ) from error - return typing.cast(JSONValue, result.model_dump(mode="json")) - - -@AgentManager.tool( - RECORD_SUPERSESSION_TOOL, - description=( - f"Record {SUPERSEDES_RELATION!r}: the successor is authorized to replace " - "the predecessor across its entire scope on the same subject." - ), -) -async def record_supersession(input: SupersessionProposal) -> JSONValue: - return await _exact_result( - SupersessionBehaviorResolver.record_supersession( - input.successor_block_id, - input.predecessor_block_id, - ) - ) - - -@AgentManager.tool( - RECORD_REFINEMENT_TOOL, - description=( - f"Record {REFINES_RELATION!r}: new compatible detail at equal or narrower scope " - "on the same subject, not mere extraction or rewording. The predecessor " - "remains independently usable as a coarser description." - ), -) -async def record_refinement(input: RefinementProposal) -> JSONValue: - return await _exact_result( - RefinementBehaviorResolver.record_refinement( - input.refinement_block_id, - input.predecessor_block_id, - ) - ) - - -@AgentManager.tool( - RECORD_EVIDENCE_STANCE_TOOL, - description=( - "Record attributable evidence supporting or challenging a whole " - "assertion in comparable scope, without declaring it true or false. " - "Merely establishing that derived content faithfully restates its source is " - "not evidence stance, regardless of source authority. Shared provenance is " - "allowed when observation or reasoning " - "contributes reasons beyond restatement." - ), -) -async def record_evidence_stance(input: EvidenceStanceProposal) -> JSONValue: - return await _exact_result( - EvidenceStanceBehaviorResolver.record_evidence_stance( - input.evidence_block_id, - input.assertion_block_id, - input.stance, - ) - ) - - -@AgentManager.tool( - CREATE_SYNTHESIS_TOOL, - description=( - "Create reusable multi-source information preserving provenance, " - "disagreement, uncertainty and speaker attribution; copies do not " - "multiply corroboration." - ), -) -async def create_synthesis(input: SynthesisProposal) -> JSONValue: - return await _exact_result( - SynthesisBehaviorResolver.create_synthesis( - input.text, - input.source_block_ids, - input.previous_synthesis_block_id, - ) - ) - - -@AgentManager.tool( - ANCHOR_EXISTING_REFERENT_TOOL, - description=( - "Link a source's referring fragment to an existing identity-bearing " - "referent, without creating a new referent." - ), -) -async def anchor_existing_referent( - input: ExistingReferentAnchorProposal, -) -> JSONValue: - return await _exact_result( - ExistingReferentAnchoringBehaviorResolver.anchor_existing_referent( - input.source_block_id, - input.selected_text, - input.referent_block_id, - ) - ) - - -@AgentManager.tool( - RECORD_DUPLICATE_ASSERTION_TOOL, - description=( - f"Record {DUPLICATES_ASSERTION_RELATION!r}: whole assertions from the same " - "provenance occurrence with no " - "independent evidence, reasoning, decision or material gain; matching " - "words alone are insufficient." - ), -) -async def record_duplicate_assertion(input: DuplicateAssertionProposal) -> JSONValue: - return await _exact_result( - DuplicateAssertionBehaviorResolver.record_duplicate_assertion( - input.left_block_id, - input.right_block_id, - ) - ) - - -@AgentManager.tool( - RECORD_ORGANIZATION_CANDIDATE_TOOL, - description="Mark an organization candidate without executing the behavior.", - input_model_factory=_candidate_input_model, -) -async def record_organization_candidate( - input: RecordOrganizationCandidateInput, -) -> JSONValue: - behavior = get_behavior_resolver(input.behavior) - if behavior is None: - raise ToolExecutionError({"error": "behavior_unavailable"}) - return await _exact_result( - typing.cast(typing.Any, behavior).record_candidate(input.block_id) - ) +if TYPE_CHECKING: + from inkcre_core.business.organization.tools import * diff --git a/app/business/organization_job.py b/app/business/organization_job.py index 270eea94..12e12c54 100644 --- a/app/business/organization_job.py +++ b/app/business/organization_job.py @@ -1,33 +1,10 @@ -"""Exact automatic Organization commands hosted by Jobs.""" +"""Legacy import; use inkcre_core for new consumers.""" -from app.business.job import JobHandler -from app.business.organization_media import ( - MEDIA_INTERPRETATION_JOB_TYPE, - can_handle_media_interpretation, - interpret_missing_media, -) -from app.schemas.job import JobModel -from app.schemas.organization import MediaInterpretationJobParameters +from typing import TYPE_CHECKING +from importlib import import_module +import sys +sys.modules[__name__] = import_module("inkcre_core.business.organization_job") -class MediaInterpretationJobHandler( - JobHandler[MediaInterpretationJobParameters], - job_type=MEDIA_INTERPRETATION_JOB_TYPE, - description="Interpret missing image, audio, and video Blocks through configured Agents.", - parameters_model=MediaInterpretationJobParameters, - default_timeout_seconds=1800, -): - @classmethod - async def can_handle(cls, parameters: MediaInterpretationJobParameters) -> bool: - del parameters - return await can_handle_media_interpretation() - - @classmethod - async def handle( - cls, - job: JobModel, - parameters: MediaInterpretationJobParameters, - ) -> None: - del parameters - report = await interpret_missing_media() - job.state = report.model_dump(mode="json") +if TYPE_CHECKING: + from inkcre_core.business.organization_job import * diff --git a/app/business/organization_media.py b/app/business/organization_media.py index a32fabc3..a3eb80d7 100644 --- a/app/business/organization_media.py +++ b/app/business/organization_media.py @@ -1,263 +1,10 @@ -"""System-driven media interpretation approach owned by Organization.""" +"""Legacy import; use inkcre_core for new consumers.""" -import json -import logging -import typing +from typing import TYPE_CHECKING +from importlib import import_module +import sys +sys.modules[__name__] = import_module("inkcre_core.business.organization_media") -from app.business.agent import AgentManager, TurnTermination -from app.business.deployment_config import DeploymentConfigManager, DeploymentConfigService -from app.business.info_base.commands import get_related_block -from app.business.info_base.resolver import ResolverManager -from app.business.info_base.resolver.audio import AudioSolvedContent -from app.business.info_base.resolver.image import ImageSolvedContent -from app.business.info_base.resolver.video import VideoSolvedContent -from app.persistence.info_base.uow import graph_uow -from app.schemas.ai import ( - AudioContentPart, - ImageContentPart, - TextContentPart, - UserContentPart, - UserMessage, - VideoContentPart, -) -from app.schemas.info_base.block import BlockModel -from app.schemas.organization import ( - MediaInterpretationConfig, - MediaInterpretationDiagnostic, - MediaInterpretationReport, -) - - -logger = logging.getLogger(__name__) - -MEDIA_INTERPRETATION_CONFIG_KEY = "core.organization.media_interpretation" -MEDIA_INTERPRETATION_CONFIG_SCHEMA = "core.organization.media_interpretation.config.v1" -MEDIA_INTERPRETATION_JOB_TYPE = "core.organization.media_interpretation.v1" -_CANDIDATE_LIMIT = 20 -_DIAGNOSTIC_LIMIT = 20 -_RESOLVER_MODALITIES: dict[str, typing.Literal["image", "audio", "video"]] = { - "core.image.v1": "image", - "core.audio.v1": "audio", - "core.video.v1": "video", -} - - -DeploymentConfigManager.register_schema( - MEDIA_INTERPRETATION_CONFIG_SCHEMA, - MediaInterpretationConfig, - keys=(MEDIA_INTERPRETATION_CONFIG_KEY,), -) - - -async def _config() -> MediaInterpretationConfig | None: - value = await DeploymentConfigService.get(MEDIA_INTERPRETATION_CONFIG_KEY) - if value is None: - return None - if not isinstance(value, MediaInterpretationConfig): - raise TypeError("Media interpretation config registry returned the wrong model") - return value - - -def _agent( - config: MediaInterpretationConfig, - modality: typing.Literal["image", "audio", "video"], -) -> int: - return { - "image": config.image_agent, - "audio": config.audio_agent, - "video": config.video_agent, - }[modality] - - -async def can_handle_media_interpretation() -> bool: - config = await _config() - if config is None: - return False - for modality in typing.cast( - tuple[typing.Literal["image", "audio", "video"], ...], - ("image", "audio", "video"), - ): - if await AgentManager.can_execute(_agent(config, modality), modality): - return True - return False - - -async def _candidates() -> tuple[BlockModel, ...]: - async with graph_uow() as uow: - return await uow.blocks.without_outgoing_relation( - tuple(_RESOLVER_MODALITIES), "interpretation", limit=_CANDIDATE_LIMIT - ) - - -async def _relation_context(block_id: int) -> list[dict[str, typing.Any]]: - async with graph_uow() as uow: - relations = (await uow.relations.get(block_id))[:20] - neighbor_ids = { - relation.to_ if relation.from_ == block_id else relation.from_ - for relation in relations - } - neighbors = {block.id: block for block in await uow.blocks.get_many(neighbor_ids)} - context: list[dict[str, typing.Any]] = [] - for relation in relations: - other_id = relation.to_ if relation.from_ == block_id else relation.from_ - other = neighbors.get(other_id) - label = None - if other is not None: - try: - label = await ResolverManager.get(other).get_label() - except Exception: - logger.debug("Could not project media neighbor label", exc_info=True) - context.append( - { - "direction": "outgoing" if relation.from_ == block_id else "incoming", - "property": relation.content, - "other_block": {"id": other_id, "label": label}, - } - ) - return context - - -async def _message(block: BlockModel) -> UserMessage | None: - resolver = ResolverManager.get(block) - solved = await resolver.get_solved_content(materialize_missing=False) - transfer_url = await resolver.get_transfer_url() - media: UserContentPart - facts: dict[str, typing.Any] - if isinstance(solved, ImageSolvedContent): - media_type = solved.detected_media_type or ( - f"image/{solved.format}" if solved.format is not None else None - ) - if media_type is None: - return None - media = ImageContentPart( - data=solved.content, - mime_type=media_type, - transfer_url=transfer_url, - ) - facts = {"format": solved.format, "width": solved.width, "height": solved.height} - elif isinstance(solved, AudioSolvedContent): - media_type = solved.detected_media_type or ( - f"audio/{solved.container}" if solved.container is not None else None - ) - if media_type is None: - return None - media = AudioContentPart( - data=solved.content, - mime_type=media_type, - transfer_url=transfer_url, - ) - facts = { - "container": solved.container, - "codec": solved.codec, - "duration_ms": solved.duration_ms, - } - elif isinstance(solved, VideoSolvedContent): - media_type = solved.detected_media_type or ( - f"video/{solved.container}" if solved.container is not None else None - ) - if media_type is None: - return None - media = VideoContentPart( - data=solved.content, - mime_type=media_type, - transfer_url=transfer_url, - ) - facts = { - "container": solved.container, - "codec": solved.video_codec, - "duration_ms": solved.duration_ms, - "width": solved.width, - "height": solved.height, - } - else: - return None - - context = { - "request": "interpret media and submit only a useful additive graph", - "focal_block": { - "id": block.id, - "resolver": block.resolver, - "facts": facts, - }, - "direct_relations": await _relation_context(typing.cast(int, block.id)), - } - return UserMessage( - content=( - TextContentPart( - text=json.dumps( - context, - ensure_ascii=False, - sort_keys=True, - separators=(",", ":"), - ) - ), - media, - ) - ) - - -async def interpret_missing_media() -> MediaInterpretationReport: - config = await _config() - if config is None: - return MediaInterpretationReport() - selected = interpreted = unavailable = failed = no_output = 0 - diagnostics: list[MediaInterpretationDiagnostic] = [] - - def diagnostic( - block: int, - modality: typing.Literal["image", "audio", "video"], - outcome: typing.Literal["unavailable", "failed", "no_output"], - reason: str, - ) -> None: - if len(diagnostics) < _DIAGNOSTIC_LIMIT: - diagnostics.append( - MediaInterpretationDiagnostic( - block=block, - modality=modality, - outcome=outcome, - reason=reason, - ) - ) - - for block in await _candidates(): - selected += 1 - block_id = typing.cast(int, block.id) - modality = _RESOLVER_MODALITIES[block.resolver] - agent = _agent(config, modality) - if not await AgentManager.can_execute(agent, modality): - unavailable += 1 - diagnostic(block_id, modality, "unavailable", "agent_not_locally_executable") - continue - try: - message = await _message(block) - if message is None: - unavailable += 1 - diagnostic(block_id, modality, "unavailable", "media_input_unavailable") - continue - thread = await AgentManager.run(agent, message) - turn = thread.current_turn - if turn is None: - raise RuntimeError("Media interpretation Agent did not start a Turn") - outcome = await turn - if outcome == TurnTermination.MAX_MODEL_CALLS: - raise RuntimeError("Media interpretation exceeded its model-call budget") - except Exception as error: - failed += 1 - logger.exception("Media interpretation failed", extra={"block": block_id}) - diagnostic(block_id, modality, "failed", type(error).__name__) - continue - if await get_related_block(block_id, content="interpretation") is None: - no_output += 1 - diagnostic(block_id, modality, "no_output", "missing_interpretation_relation") - else: - interpreted += 1 - - return MediaInterpretationReport( - selected=selected, - interpreted=interpreted, - unavailable=unavailable, - failed=failed, - no_output=no_output, - diagnostics=tuple(diagnostics), - ) +if TYPE_CHECKING: + from inkcre_core.business.organization_media import * diff --git a/app/business/peer/__init__.py b/app/business/peer/__init__.py index 1be40f31..bc9d585e 100644 --- a/app/business/peer/__init__.py +++ b/app/business/peer/__init__.py @@ -1,26 +1,3 @@ -"""Peer discovery and request-response capability delegation.""" +"""Legacy import; use inkcre_core for new consumers.""" -from .contracts import ( - CapabilityDelegationUnavailable, - DuplicatePeerRegistrationError, - PeerError, - PeerOutcomeUnknown, - PeerProtocolConfigurationError, - PeerProtocolError, - PeerRequestNotExecuted, -) -from .http import PeerHTTPInbound, PeerHTTPOutbound -from .main import PeerManager - -__all__ = [ - "CapabilityDelegationUnavailable", - "DuplicatePeerRegistrationError", - "PeerError", - "PeerHTTPInbound", - "PeerHTTPOutbound", - "PeerManager", - "PeerOutcomeUnknown", - "PeerProtocolConfigurationError", - "PeerProtocolError", - "PeerRequestNotExecuted", -] +from inkcre_core.business.peer import * diff --git a/app/business/peer/contracts.py b/app/business/peer/contracts.py index 2ccf0490..da841cba 100644 --- a/app/business/peer/contracts.py +++ b/app/business/peer/contracts.py @@ -1,68 +1,10 @@ -"""Peer discovery, routing, and outbound failure contracts.""" +"""Legacy import; use inkcre_core for new consumers.""" -from collections.abc import Callable -from dataclasses import dataclass -import typing +from typing import TYPE_CHECKING +from importlib import import_module +import sys -from app.schemas.ai import JSONValue -from app.schemas.peer import ( - CapabilityID, - CorePeerConfig, - PeerCapabilityAdvertisement, - PeerModel, - PeerProtocolID, -) +sys.modules[__name__] = import_module("inkcre_core.business.peer.contracts") - -class PeerError(RuntimeError): - """Base failure exposed by the Peer domain.""" - - -class DuplicatePeerRegistrationError(PeerError): - pass - - -class CapabilityDelegationUnavailable(PeerError): - pass - - -class PeerProtocolError(PeerError): - pass - - -class PeerProtocolConfigurationError(PeerProtocolError): - """One advertised inbound cannot construct its caller-local outbound.""" - - -class PeerRequestNotExecuted(PeerProtocolError): - """The selected outbound proved capability execution did not begin.""" - - -class PeerOutcomeUnknown(PeerProtocolError): - """Dispatch may have occurred but no execution outcome is known.""" - - -class PeerInbound(typing.Protocol): - @property - def capability(self) -> CapabilityID: ... - - def advertise( - self, - config: CorePeerConfig, - ) -> PeerCapabilityAdvertisement | None: ... - - -class PeerOutbound(typing.Protocol): - async def execute(self, payload: JSONValue) -> JSONValue: ... - - -PeerOutboundFactory: typing.TypeAlias = Callable[ - [PeerModel, dict[str, JSONValue]], - PeerOutbound, -] - - -@dataclass(frozen=True) -class PeerOutboundRegistration: - protocol: PeerProtocolID - factory: PeerOutboundFactory +if TYPE_CHECKING: + from inkcre_core.business.peer.contracts import * diff --git a/app/business/peer/http.py b/app/business/peer/http.py index 6a3a7c0d..af1bbd27 100644 --- a/app/business/peer/http.py +++ b/app/business/peer/http.py @@ -1,202 +1,10 @@ -"""core.peer.protocol.http.v1 inbound advertisement and one-shot outbound.""" +"""Legacy import; use inkcre_core for new consumers.""" -from __future__ import annotations +from typing import TYPE_CHECKING +from importlib import import_module +import sys -from collections import defaultdict -from dataclasses import dataclass -import json -import typing +sys.modules[__name__] = import_module("inkcre_core.business.peer.http") -import httpx -import pydantic - -from app.middleware import create_peer_jwt -from app.schemas.ai import JSONValue -from app.schemas.peer import ( - PEER_EXECUTION_HEADER, - PEER_HTTP_PROTOCOL, - PEER_NOT_EXECUTED, - CapabilityID, - CorePeerConfig, - PeerCapabilityAdvertisement, - PeerHTTPInboundParameters, - PeerInboundInterface, - PeerModel, - PeerProtocolRequest, - PeerProtocolResponse, -) -from app.settings import settings -from app.observability import inject_context -from libs.obsrv.telemetry import operation -from opentelemetry.trace import SpanKind - -from .contracts import ( - PeerOutcomeUnknown, - PeerProtocolConfigurationError, - PeerProtocolError, - PeerRequestNotExecuted, -) - - -_RESERVED_REQUEST_HEADERS = frozenset( - { - "authorization", - "connection", - "content-length", - "host", - "keep-alive", - "proxy-authenticate", - "proxy-authorization", - "te", - "trailer", - "transfer-encoding", - "upgrade", - } -) - - -@dataclass(frozen=True) -class PeerHTTPInbound: - """One fixed Business route advertised through Peer HTTP v1.""" - - capability: CapabilityID - method: str - path: str - - def __post_init__(self) -> None: - if not self.path.startswith("/") or "?" in self.path or "#" in self.path: - raise ValueError("Peer HTTP inbound path must be one absolute path") - - def advertise( - self, - config: CorePeerConfig, - ) -> PeerCapabilityAdvertisement | None: - if config.http_public_base_url is None: - return None - parameters = PeerHTTPInboundParameters( - method=self.method, - url=f"{config.http_public_base_url}{self.path}", - ) - return PeerCapabilityAdvertisement( - id=self.capability, - inbound=PeerInboundInterface( - protocol=PEER_HTTP_PROTOCOL, - parameters=typing.cast( - dict[str, JSONValue], - parameters.model_dump(mode="json"), - ), - ), - ) - - -class PeerHTTPOutbound: - """One-shot authenticated JSON HTTP execution for one advertised inbound.""" - - def __init__( - self, - peer: PeerModel, - parameters: dict[str, JSONValue], - ) -> None: - self.peer = peer - try: - self.parameters = PeerHTTPInboundParameters.model_validate(parameters) - except pydantic.ValidationError as error: - raise PeerProtocolConfigurationError( - f"Peer {peer.id} published invalid HTTP inbound parameters" - ) from error - - async def execute(self, payload: JSONValue) -> JSONValue: - try: - request = PeerProtocolRequest.model_validate(payload) - except pydantic.ValidationError as error: - raise PeerProtocolError("Invalid Peer HTTP request envelope") from error - - reserved = _RESERVED_REQUEST_HEADERS.intersection(request.headers) - if reserved: - rendered = ", ".join(sorted(reserved)) - raise PeerProtocolError(f"Peer HTTP payload attempted reserved headers: {rendered}") - - query = [(name, value) for name, values in request.query.items() for value in values] - headers = [ - (name, value) for name, values in request.headers.items() for value in values - ] - headers.append( - ( - "authorization", - f"Bearer {create_peer_jwt(settings.jwt_secret)}", - ) - ) - kwargs: dict[str, typing.Any] = { - "method": self.parameters.method, - "url": self.parameters.url, - "params": query, - "headers": headers, - } - if "body" in request.model_fields_set: - kwargs["json"] = request.body - - with operation( - "peer.http", - attributes={"inkcre.peer.target_id": str(self.peer.id)}, - kind=SpanKind.CLIENT, - ) as observation: - span = observation.span - propagated = inject_context() - if propagated: - # The transport owns Trace Context; business payload headers cannot duplicate it. - headers[:] = [ - (key, value) - for key, value in headers - if key.lower() not in {"traceparent", "tracestate"} - ] - headers.extend(propagated.items()) - try: - async with httpx.AsyncClient( - timeout=settings.peer_http_timeout_seconds, - ) as client: - response = await client.request(**kwargs) - except (httpx.ConnectError, httpx.ConnectTimeout, httpx.PoolTimeout) as error: - span.set_attribute("inkcre.peer.outcome", "not_executed") - raise PeerRequestNotExecuted( - f"Could not dispatch to Peer {self.peer.id}" - ) from error - except httpx.RequestError as error: - span.set_attribute("inkcre.peer.outcome", "unknown") - raise PeerOutcomeUnknown( - f"Peer {self.peer.id} dispatch outcome is unknown" - ) from error - - if ( - response.headers.get(PEER_EXECUTION_HEADER, "").strip().lower() == PEER_NOT_EXECUTED - ): - span.set_attribute("inkcre.peer.outcome", "not_executed") - raise PeerRequestNotExecuted( - f"Peer {self.peer.id} reported that execution did not begin" - ) - - span.set_attribute("inkcre.peer.outcome", "responded") - span.set_attribute("http.response.status_code", response.status_code) - if response.status_code >= 500: - observation.outcome = "error" - - grouped_headers: defaultdict[str, list[str]] = defaultdict(list) - for name, value in response.headers.multi_items(): - if name.lower() != PEER_EXECUTION_HEADER.lower(): - grouped_headers[name.lower()].append(value) - - response_fields: dict[str, typing.Any] = { - "status": response.status_code, - "headers": grouped_headers, - } - if response.content: - try: - response_fields["body"] = response.json() - except json.JSONDecodeError as error: - raise PeerProtocolError( - f"Peer {self.peer.id} returned a non-JSON HTTP body" - ) from error - normalized = PeerProtocolResponse.model_validate(response_fields) - return typing.cast( - JSONValue, - normalized.model_dump(mode="json", exclude_unset=True), - ) +if TYPE_CHECKING: + from inkcre_core.business.peer.http import * diff --git a/app/business/peer/main.py b/app/business/peer/main.py index 7de8f379..73ceec71 100644 --- a/app/business/peer/main.py +++ b/app/business/peer/main.py @@ -1,253 +1,10 @@ -"""Peer identity, local inbound publication, discovery, and delegation.""" +"""Legacy import; use inkcre_core for new consumers.""" -from __future__ import annotations +from typing import TYPE_CHECKING +from importlib import import_module +import sys -from dataclasses import dataclass -import datetime -import logging -import random -import typing +sys.modules[__name__] = import_module("inkcre_core.business.peer.main") -import pydantic - -from app.configuration import ConfigContract -from app.schemas.ai import JSONValue -from app.schemas.peer import ( - PEER_HTTP_PROTOCOL, - CapabilityID, - CorePeerConfig, - PeerCapabilityAdvertisement, - PeerModel, - PeerRef, - normalize_capability_snapshot, -) -from app.settings import settings -from app.version import APPLICATION_VERSION - -from .contracts import ( - CapabilityDelegationUnavailable, - DuplicatePeerRegistrationError, - PeerInbound, - PeerOutcomeUnknown, - PeerOutboundFactory, - PeerProtocolConfigurationError, - PeerRequestNotExecuted, -) -from .http import PeerHTTPOutbound -from app.persistence.peer.uow import peer_uow - - -logger = logging.getLogger(__name__) - - -@dataclass(frozen=True) -class _Candidate: - peer: PeerModel - advertisement: PeerCapabilityAdvertisement - - -class PeerManager: - """Own Peer facts, local inbound registry, discovery, and one-shot routing.""" - - _INBOUNDS: dict[CapabilityID, PeerInbound] = {} - _OUTBOUNDS: dict[str, PeerOutboundFactory] = {} - _random: random.Random = random.SystemRandom() - _config_contract = ConfigContract(CorePeerConfig) - - @classmethod - async def register_self(cls) -> PeerModel: - """Upsert runtime-owned identity/schema without changing owner configuration.""" - async with peer_uow() as peers: - peer = await peers.register( - settings.peer_id, - settings.peer_name, - APPLICATION_VERSION, - cls._config_contract.json_schema(), - ) - logger.info("Peer registered", extra={"peer": str(peer.id), "name": peer.name}) - return peer - - @classmethod - def get_current_peer_ref(cls) -> PeerRef: - return settings.peer_id - - @classmethod - async def get_current_config_async(cls) -> CorePeerConfig: - peer = await cls.get_async(cls.get_current_peer_ref()) - if peer is None: - raise RuntimeError("Current Peer must be registered before reading config") - try: - return cls._config_contract.validate(peer.config) - except pydantic.ValidationError as error: - raise ValueError("Current Peer config is invalid") from error - - @classmethod - async def get_async(cls, peer: PeerRef) -> PeerModel | None: - async with peer_uow() as peers: - return await peers.get(peer) - - @classmethod - async def get_all(cls) -> tuple[PeerModel, ...]: - async with peer_uow() as peers: - return await peers.get_all() - - @classmethod - async def get_with_lease(cls, peer: PeerRef) -> dict[str, typing.Any] | None: - async with peer_uow() as peers: - rows, _ = await peers.with_leases(peer_id=peer) - return rows[0] if rows else None - - @classmethod - async def list_with_leases( - cls, *, limit: int | None = None, cursor: PeerRef | None = None - ) -> tuple[list[dict[str, typing.Any]], PeerRef | None]: - async with peer_uow() as peers: - return await peers.with_leases(limit=limit, cursor=cursor) - - @classmethod - def register_inbound(cls, inbound: PeerInbound) -> bool: - """Register one inbound and report whether this call added it.""" - existing = cls._INBOUNDS.get(inbound.capability) - if existing is inbound or existing == inbound: - return False - if existing is not None: - raise DuplicatePeerRegistrationError( - f"Capability inbound {inbound.capability!r} is already registered" - ) - cls._INBOUNDS[inbound.capability] = inbound - return True - - @classmethod - def unregister_inbound(cls, capability: CapabilityID) -> None: - cls._INBOUNDS.pop(capability, None) - - @classmethod - def register_outbound( - cls, - protocol: str, - factory: PeerOutboundFactory, - ) -> None: - existing = cls._OUTBOUNDS.get(protocol) - if existing is factory: - return - if existing is not None: - raise DuplicatePeerRegistrationError( - f"Peer outbound {protocol!r} is already registered" - ) - cls._OUTBOUNDS[protocol] = factory - - @classmethod - def setup_builtin_outbounds(cls) -> None: - cls.register_outbound(PEER_HTTP_PROTOCOL, PeerHTTPOutbound) - - @classmethod - async def publish_self(cls) -> PeerModel: - """Replace this Peer-owned capability snapshot from current config/registry.""" - async with peer_uow() as peers: - peer = await peers.get(settings.peer_id, for_update=True) - if peer is None: - raise RuntimeError("Current Peer must be registered before publication") - try: - config = cls._config_contract.validate(peer.config) - except pydantic.ValidationError as error: - raise ValueError("Current Peer config is invalid") from error - advertisements = normalize_capability_snapshot( - advertisement - for inbound in cls._INBOUNDS.values() - if (advertisement := inbound.advertise(config)) is not None - ) - normalized = [ - typing.cast(dict[str, JSONValue], advertisement.model_dump(mode="json")) - for advertisement in advertisements - ] - if peer.capabilities != normalized: - peer.capabilities = normalized - await peers.save(peer) - return peer - - @classmethod - async def renew_self_lease(cls, ttl_seconds: int) -> datetime.datetime: - """Renew liveness through the database-time helper.""" - if ttl_seconds <= 0: - raise ValueError("Peer lease TTL must be positive") - async with peer_uow() as peers: - expiry = await peers.renew_lease(settings.peer_id, ttl_seconds) - return expiry - - @classmethod - async def refresh_self(cls, ttl_seconds: int) -> PeerModel: - """Refresh advertisement, then renew its route lease.""" - peer = await cls.publish_self() - peer.lease_expires_at = await cls.renew_self_lease(ttl_seconds) - return peer - - @classmethod - async def clear_self_lease(cls) -> None: - async with peer_uow() as peers: - await peers.clear_lease(settings.peer_id) - - @classmethod - async def delegate( - cls, - capability: CapabilityID, - payload: JSONValue, - *, - route_to_peer: PeerRef | None = None, - ) -> JSONValue: - """Execute through one eligible Peer, failing over only after non-execution.""" - candidates = await cls._candidates(capability, route_to_peer) - attempted = 0 - for candidate in candidates: - factory = cls._OUTBOUNDS.get(candidate.advertisement.inbound.protocol) - if factory is None: # candidate eligibility is rechecked defensively - continue - try: - outbound = factory( - candidate.peer, - candidate.advertisement.inbound.parameters, - ) - except PeerProtocolConfigurationError: - continue - attempted += 1 - try: - return await outbound.execute(payload) - except PeerRequestNotExecuted: - if route_to_peer is not None: - break - continue - except PeerOutcomeUnknown: - raise - raise CapabilityDelegationUnavailable( - f"No eligible Peer completed capability {capability!r}; attempted={attempted}" - ) - - @classmethod - async def _candidates( - cls, - capability: CapabilityID, - route_to_peer: PeerRef | None, - ) -> tuple[_Candidate, ...]: - async with peer_uow() as peers_repository: - peers = await peers_repository.candidates(settings.peer_id, route_to_peer) - - candidates: list[_Candidate] = [] - for peer in peers: - try: - snapshots = peer.capability_snapshot() - except (pydantic.ValidationError, ValueError): - logger.warning( - "Skipping malformed Peer capability snapshot", - extra={"peer": str(peer.id)}, - ) - continue - for advertisement in snapshots: - if ( - advertisement.id == capability - and advertisement.inbound.protocol in cls._OUTBOUNDS - ): - candidates.append(_Candidate(peer, advertisement)) - break - - if route_to_peer is None: - cls._random.shuffle(candidates) - return tuple(candidates) +if TYPE_CHECKING: + from inkcre_core.business.peer.main import * diff --git a/app/business/semantic_retrieval/__init__.py b/app/business/semantic_retrieval/__init__.py index c6a14424..b8e9adb9 100644 --- a/app/business/semantic_retrieval/__init__.py +++ b/app/business/semantic_retrieval/__init__.py @@ -1,35 +1,3 @@ -"""Profile-scoped local semantic retrieval and embedding maintenance.""" +"""Legacy import; use inkcre_core for new consumers.""" -from .main import ( - EmbeddingProfileNotFoundError, - InvalidSemanticVectorError, - SEMANTIC_RETRIEVAL_CONFIG_KEY, - SEMANTIC_RETRIEVAL_CONFIG_SCHEMA, - SEMANTIC_RETRIEVAL_CAPABILITY, - SemanticRetrievalDelegationError, - SemanticRetrievalError, - SemanticRetrievalManager, - SemanticRetrievalNotConfiguredError, -) -from .job import ( - SEMANTIC_MAINTAIN_JOB_TYPE, - SEMANTIC_REBUILD_JOB_TYPE, - SemanticMaintainJobHandler, - SemanticRebuildJobHandler, -) - -__all__ = [ - "EmbeddingProfileNotFoundError", - "InvalidSemanticVectorError", - "SEMANTIC_RETRIEVAL_CONFIG_KEY", - "SEMANTIC_RETRIEVAL_CONFIG_SCHEMA", - "SEMANTIC_RETRIEVAL_CAPABILITY", - "SemanticRetrievalDelegationError", - "SemanticRetrievalError", - "SemanticRetrievalManager", - "SemanticRetrievalNotConfiguredError", - "SEMANTIC_MAINTAIN_JOB_TYPE", - "SEMANTIC_REBUILD_JOB_TYPE", - "SemanticMaintainJobHandler", - "SemanticRebuildJobHandler", -] +from inkcre_core.business.semantic_retrieval import * diff --git a/app/business/semantic_retrieval/job.py b/app/business/semantic_retrieval/job.py index fc0b5a0f..21e0142c 100644 --- a/app/business/semantic_retrieval/job.py +++ b/app/business/semantic_retrieval/job.py @@ -1,59 +1,10 @@ -"""Exact semantic embedding maintenance commands hosted by Jobs.""" +"""Legacy import; use inkcre_core for new consumers.""" -from app.business.job import JobHandler -from app.schemas.job import JobModel -from app.schemas.semantic_retrieval import EmbeddingMaintenanceJobParameters +from typing import TYPE_CHECKING +from importlib import import_module +import sys -from .main import SemanticRetrievalManager +sys.modules[__name__] = import_module("inkcre_core.business.semantic_retrieval.job") - -SEMANTIC_MAINTAIN_JOB_TYPE = "core.semantic_retrieval.maintain.v1" -SEMANTIC_REBUILD_JOB_TYPE = "core.semantic_retrieval.rebuild.v1" - - -class SemanticMaintainJobHandler( - JobHandler[EmbeddingMaintenanceJobParameters], - job_type=SEMANTIC_MAINTAIN_JOB_TYPE, - description="Maintain missing or stale semantic embedding records.", - parameters_model=EmbeddingMaintenanceJobParameters, - default_timeout_seconds=900, -): - @classmethod - async def can_handle(cls, parameters: EmbeddingMaintenanceJobParameters) -> bool: - return await SemanticRetrievalManager.can_maintain(parameters.profile) - - @classmethod - async def handle( - cls, - job: JobModel, - parameters: EmbeddingMaintenanceJobParameters, - ) -> None: - report = await SemanticRetrievalManager.maintain( - parameters.profile, - parameters.options, - ) - job.state = report.model_dump(mode="json") - - -class SemanticRebuildJobHandler( - JobHandler[EmbeddingMaintenanceJobParameters], - job_type=SEMANTIC_REBUILD_JOB_TYPE, - description="Rebuild semantic embedding records present before invocation.", - parameters_model=EmbeddingMaintenanceJobParameters, - default_timeout_seconds=1800, -): - @classmethod - async def can_handle(cls, parameters: EmbeddingMaintenanceJobParameters) -> bool: - return await SemanticRetrievalManager.can_maintain(parameters.profile) - - @classmethod - async def handle( - cls, - job: JobModel, - parameters: EmbeddingMaintenanceJobParameters, - ) -> None: - report = await SemanticRetrievalManager.rebuild( - parameters.profile, - parameters.options, - ) - job.state = report.model_dump(mode="json") +if TYPE_CHECKING: + from inkcre_core.business.semantic_retrieval.job import * diff --git a/app/business/semantic_retrieval/main.py b/app/business/semantic_retrieval/main.py index 709925e7..487eee50 100644 --- a/app/business/semantic_retrieval/main.py +++ b/app/business/semantic_retrieval/main.py @@ -1,548 +1,10 @@ -"""Profile-scoped embedding maintenance and exact semantic retrieval.""" +"""Legacy import; use inkcre_core for new consumers.""" -from __future__ import annotations +from typing import TYPE_CHECKING +from importlib import import_module +import sys -from dataclasses import dataclass -import datetime -import logging -import math -import typing +sys.modules[__name__] = import_module("inkcre_core.business.semantic_retrieval.main") -import pydantic - -from app.business.ai import AIExecutionRequirement, AIManager -from app.business.deployment_config import DeploymentConfigManager, DeploymentConfigService -from app.business.info_base.services import RelationService -from app.business.info_base.resolver import ( - ResolverManager, - UnknownResolverError, - UnsupportedResolverCapability, -) -from app.business.peer import PeerManager -from app.persistence.semantic_retrieval.uow import semantic_retrieval_uow -from app.schemas.ai import ( - BlockEmbeddingModel, - EmbeddingProfileID, - EmbeddingProfileModel, - RelationEmbeddingModel, -) -from app.schemas.info_base.block import BlockModel -from app.schemas.info_base.relation import RelationModel -from app.schemas.peer import PeerProtocolRequest, PeerProtocolResponse, PeerRef -from app.schemas.semantic_retrieval import ( - BlockSemanticRetrievalMatch, - EmbeddingMaintenanceDiagnostic, - EmbeddingMaintenanceOptions, - EmbeddingMaintenanceReport, - RelationSemanticRetrievalMatch, - SemanticEntityType, - SemanticRetrievalConfig, - SemanticRetrievalMatch, - SemanticRetrievalRequest, - SemanticRetrievalResult, - VectorRetrievalOptions, -) - - -logger = logging.getLogger(__name__) - -SEMANTIC_RETRIEVAL_CONFIG_KEY = "semantic_retrieval" -SEMANTIC_RETRIEVAL_CONFIG_SCHEMA = "core.semantic_retrieval.config.v1" -SEMANTIC_RETRIEVAL_CAPABILITY = "core.semantic_retrieval.v1" - - -class SemanticRetrievalError(RuntimeError): - pass - - -class SemanticRetrievalNotConfiguredError(SemanticRetrievalError): - pass - - -class EmbeddingProfileNotFoundError(SemanticRetrievalError): - pass - - -class InvalidSemanticVectorError(SemanticRetrievalError): - pass - - -class SemanticRetrievalDelegationError(SemanticRetrievalError): - pass - - -class _ProjectionUnavailable(Exception): - pass - - -@dataclass(frozen=True) -class _Candidate: - entity_type: SemanticEntityType - entity_id: int - entity: BlockModel | RelationModel - text: str - - -@dataclass(frozen=True) -class _CandidatePage: - entities: tuple[BlockModel | RelationModel, ...] - next_cursor: int - exhausted: bool - - -@dataclass -class _ReportBuilder: - profile: EmbeddingProfileID - diagnostic_limit: int - embedded: int = 0 - unavailable: int = 0 - failed: int = 0 - diagnostics: list[EmbeddingMaintenanceDiagnostic] | None = None - - def __post_init__(self) -> None: - self.diagnostics = [] - - def record( - self, - entity_type: SemanticEntityType, - entity: int, - outcome: typing.Literal["unavailable", "failed"], - reason: str, - ) -> None: - if outcome == "unavailable": - self.unavailable += 1 - else: - self.failed += 1 - diagnostics = typing.cast(list[EmbeddingMaintenanceDiagnostic], self.diagnostics) - if len(diagnostics) < self.diagnostic_limit: - diagnostics.append( - EmbeddingMaintenanceDiagnostic( - entity_type=entity_type, - entity=entity, - outcome=outcome, - reason=reason, - ) - ) - - def build(self) -> EmbeddingMaintenanceReport: - return EmbeddingMaintenanceReport( - profile=self.profile, - embedded=self.embedded, - unavailable=self.unavailable, - failed=self.failed, - diagnostics=tuple(self.diagnostics or ()), - ) - - -DeploymentConfigManager.register_schema( - SEMANTIC_RETRIEVAL_CONFIG_SCHEMA, - SemanticRetrievalConfig, - keys=(SEMANTIC_RETRIEVAL_CONFIG_KEY,), -) - - -class SemanticRetrievalManager: - """Single use-domain owner for projection, records, ranking and defaults.""" - - @classmethod - async def get_profile( - cls, profile_id: EmbeddingProfileID - ) -> EmbeddingProfileModel | None: - async with semantic_retrieval_uow() as repository: - return await repository.get_profile(profile_id) - - @classmethod - async def list_profiles(cls, *, limit: int | None = None, cursor: int | None = None): - async with semantic_retrieval_uow() as repository: - return await repository.list_profiles(limit=limit, cursor=cursor) - - @classmethod - async def _configured_profile_id(cls) -> EmbeddingProfileID: - value = await DeploymentConfigService.get(SEMANTIC_RETRIEVAL_CONFIG_KEY) - if value is None: - raise SemanticRetrievalNotConfiguredError( - "Semantic retrieval default Profile is not configured" - ) - if not isinstance(value, SemanticRetrievalConfig): - raise TypeError("Semantic retrieval config registry returned the wrong model") - if value.default_profile is None: - raise SemanticRetrievalNotConfiguredError( - "Semantic retrieval default Profile is not configured" - ) - return value.default_profile - - @classmethod - async def _load_profile( - cls, - profile: EmbeddingProfileID | None, - ) -> EmbeddingProfileModel: - profile_id = await cls._configured_profile_id() if profile is None else profile - async with semantic_retrieval_uow() as repository: - result = await repository.get_profile(profile_id) - if result is None: - raise EmbeddingProfileNotFoundError(f"Embedding Profile {profile_id} does not exist") - return result - - @classmethod - async def can_maintain(cls, profile: EmbeddingProfileID | None = None) -> bool: - """Return static local Job eligibility without probing the provider.""" - try: - selected = await cls._load_profile(profile) - except (SemanticRetrievalNotConfiguredError, EmbeddingProfileNotFoundError): - return False - return await AIManager.can_execute( - selected.ai_model, - AIExecutionRequirement( - capability="embedding", - input_modalities=frozenset({"text"}), - output_modalities=frozenset({"vector"}), - ), - ) - - @classmethod - async def retrieve( - cls, - query: str, - profile: EmbeddingProfileID | None = None, - options: VectorRetrievalOptions | None = None, - *, - route_to_peer: PeerRef | None = None, - ) -> SemanticRetrievalResult: - """Execute locally unless the caller explicitly selects another Peer.""" - request = SemanticRetrievalRequest( - query=query, - profile=profile, - options=options or VectorRetrievalOptions(), - ) - if route_to_peer is None or route_to_peer == PeerManager.get_current_peer_ref(): - return await cls.retrieve_local( - request.query, - request.profile, - request.options, - ) - - payload = PeerProtocolRequest( - body=typing.cast( - typing.Any, - request.model_dump(mode="json"), - ) - ) - result = await PeerManager.delegate( - SEMANTIC_RETRIEVAL_CAPABILITY, - typing.cast(typing.Any, payload.model_dump(mode="json", exclude_unset=True)), - route_to_peer=route_to_peer, - ) - try: - response = PeerProtocolResponse.model_validate(result) - if response.status != 200 or "body" not in response.model_fields_set: - raise SemanticRetrievalDelegationError( - f"Semantic retrieval Peer returned HTTP {response.status}" - ) - return SemanticRetrievalResult.model_validate(response.body) - except pydantic.ValidationError as error: - raise SemanticRetrievalDelegationError( - "Semantic retrieval Peer returned an invalid response" - ) from error - - @classmethod - async def retrieve_local( - cls, - query: str, - profile: EmbeddingProfileID | None = None, - options: VectorRetrievalOptions | None = None, - ) -> SemanticRetrievalResult: - """Execute exact local retrieval without entering Peer delegation.""" - if not query.strip(): - raise ValueError("query must not be empty") - retrieval_options = options or VectorRetrievalOptions() - selected = await cls._load_profile(profile) - profile_id = cls._profile_id(selected) - query_vector = ( - await AIManager.embed( - selected.ai_model, - (query,), - selected.dimensions, - ) - )[0] - cls._require_non_zero_vector(query_vector) - - matches: list[SemanticRetrievalMatch] = [] - async with semantic_retrieval_uow() as repository: - if "block" in retrieval_options.entity_types: - rows = await repository.retrieve_blocks(selected, query_vector, retrieval_options) - matches.extend( - BlockSemanticRetrievalMatch(entity=block, score=cls._score(distance)) - for block, distance in rows - ) - if "relation" in retrieval_options.entity_types: - rows = await repository.retrieve_relations( - selected, query_vector, retrieval_options - ) - matches.extend( - RelationSemanticRetrievalMatch(entity=relation, score=cls._score(distance)) - for relation, distance in rows - ) - - matches.sort( - key=lambda match: ( - -match.score, - match.type, - typing.cast(int, match.entity.id), - ) - ) - return SemanticRetrievalResult( - profile=profile_id, - matches=tuple(matches[: retrieval_options.limit]), - ) - - @classmethod - async def maintain( - cls, - profile: EmbeddingProfileID | None = None, - options: EmbeddingMaintenanceOptions | None = None, - ) -> EmbeddingMaintenanceReport: - """Embed only missing/stale records and resume through durable successes.""" - selected = await cls._load_profile(profile) - return await cls._maintain( - selected, - options or EmbeddingMaintenanceOptions(), - rebuild_cutoff=None, - ) - - @classmethod - async def rebuild( - cls, - profile: EmbeddingProfileID | None = None, - options: EmbeddingMaintenanceOptions | None = None, - ) -> EmbeddingMaintenanceReport: - """Re-embed records that were present before this invocation began.""" - selected = await cls._load_profile(profile) - async with semantic_retrieval_uow() as repository: - cutoff = await repository.database_now() - return await cls._maintain( - selected, - options or EmbeddingMaintenanceOptions(), - rebuild_cutoff=cutoff, - ) - - @classmethod - async def maintain_default( - cls, - options: EmbeddingMaintenanceOptions, - ) -> EmbeddingMaintenanceReport | None: - """Scheduled no-op when this deployment has no selected default Profile.""" - try: - return await cls.maintain(options=options) - except SemanticRetrievalNotConfiguredError: - return None - - @classmethod - async def _maintain( - cls, - profile: EmbeddingProfileModel, - options: EmbeddingMaintenanceOptions, - *, - rebuild_cutoff: datetime.datetime | None, - ) -> EmbeddingMaintenanceReport: - profile_id = cls._profile_id(profile) - report = _ReportBuilder(profile_id, options.diagnostic_limit) - batch: list[_Candidate] = [] - - async def offer(entity_type: SemanticEntityType, entity) -> bool: - entity_id = typing.cast(int | None, entity.id) - if entity_id is None: - raise RuntimeError("Persisted semantic candidate is missing its ID") - try: - text = await cls._project(entity_type, entity) - except _ProjectionUnavailable as error: - report.record(entity_type, entity_id, "unavailable", str(error)) - return True - except Exception as error: - logger.exception( - "Semantic projection failed", - extra={"entity_type": entity_type, "entity": entity_id}, - ) - report.record(entity_type, entity_id, "failed", type(error).__name__) - return True - batch.append(_Candidate(entity_type, entity_id, entity, text)) - if ( - len(batch) >= options.batch_size - or report.embedded + len(batch) >= options.max_embeddings - ): - return await flush() - return True - - async def flush() -> bool: - if not batch: - return True - current = tuple(batch) - batch.clear() - try: - await cls._embed_and_upsert(profile, current) - except Exception as error: - logger.exception( - "Semantic embedding batch failed", - extra={"profile": profile_id, "batch_size": len(current)}, - ) - for candidate in current: - report.record( - candidate.entity_type, - candidate.entity_id, - "failed", - type(error).__name__, - ) - return False - report.embedded += len(current) - return report.embedded < options.max_embeddings - - for entity_type in typing.cast(tuple[SemanticEntityType, ...], ("block", "relation")): - cursor = 0 - while report.embedded + len(batch) < options.max_embeddings: - page = await cls._candidate_page( - profile, - entity_type, - cursor, - options.scan_page_size, - rebuild_cutoff, - ) - cursor = page.next_cursor - for entity in page.entities: - if not await offer(entity_type, entity): - return report.build() - if report.embedded >= options.max_embeddings: - return report.build() - if page.exhausted: - break - await flush() - return report.build() - - @classmethod - async def _project( - cls, - entity_type: SemanticEntityType, - entity: BlockModel | RelationModel, - ) -> str: - try: - if entity_type == "block": - text = await ResolverManager.get(typing.cast(BlockModel, entity)).get_text() - else: - text = await RelationService.get_text(typing.cast(RelationModel, entity)) - except UnknownResolverError as error: - raise _ProjectionUnavailable("unknown_resolver") from error - except UnsupportedResolverCapability as error: - raise _ProjectionUnavailable("unsupported_text") from error - if text is None or not text.strip(): - raise _ProjectionUnavailable("empty_projection") - return text - - @classmethod - async def _embed_and_upsert( - cls, - profile: EmbeddingProfileModel, - candidates: tuple[_Candidate, ...], - ) -> None: - vectors = await AIManager.embed( - profile.ai_model, - tuple(candidate.text for candidate in candidates), - profile.dimensions, - ) - for vector in vectors: - cls._require_non_zero_vector(vector) - - profile_id = cls._profile_id(profile) - blocks = [] - relations = [] - for candidate, vector in zip(candidates, vectors, strict=True): - row = { - "profile": profile_id, - candidate.entity_type: candidate.entity_id, - "embedding": vector, - } - (blocks if candidate.entity_type == "block" else relations).append(row) - async with semantic_retrieval_uow() as repository: - await repository.upsert(blocks, relations) - - @classmethod - async def _candidate_page( - cls, - profile: EmbeddingProfileModel, - entity_type: SemanticEntityType, - cursor: int, - page_size: int, - rebuild_cutoff: datetime.datetime | None, - ) -> _CandidatePage: - profile_id = cls._profile_id(profile) - async with semantic_retrieval_uow() as repository: - if entity_type == "block": - rows = await repository.candidate_blocks(profile_id, cursor, page_size) - entities = tuple( - block - for block, record in rows - if cls._block_requires_embedding(profile, block, record, rebuild_cutoff) - ) - else: - rows = await repository.candidate_relations(profile_id, cursor, page_size) - entities = tuple( - relation - for relation, record, from_updated, to_updated in rows - if cls._relation_requires_embedding( - profile, relation, record, (from_updated, to_updated), rebuild_cutoff - ) - ) - next_cursor = typing.cast(int, rows[-1][0].id) if rows else cursor - return _CandidatePage(entities, next_cursor, len(rows) < page_size) - - @staticmethod - def _block_requires_embedding( - profile: EmbeddingProfileModel, - block: BlockModel, - record: BlockEmbeddingModel | None, - rebuild_cutoff: datetime.datetime | None, - ) -> bool: - if record is None: - return True - if rebuild_cutoff is not None: - return record.updated_at < rebuild_cutoff - return ( - len(record.embedding) != profile.dimensions - or record.updated_at < profile.updated_at - or record.updated_at < block.updated_at - ) - - @staticmethod - def _relation_requires_embedding( - profile: EmbeddingProfileModel, - relation: RelationModel, - record: RelationEmbeddingModel | None, - endpoint_updated_at: tuple[datetime.datetime, datetime.datetime], - rebuild_cutoff: datetime.datetime | None, - ) -> bool: - if record is None: - return True - if rebuild_cutoff is not None: - return record.updated_at < rebuild_cutoff - return ( - len(record.embedding) != profile.dimensions - or record.updated_at < profile.updated_at - or record.updated_at < relation.updated_at - or record.updated_at < endpoint_updated_at[0] - or record.updated_at < endpoint_updated_at[1] - ) - - @staticmethod - def _profile_id(profile: EmbeddingProfileModel) -> EmbeddingProfileID: - if profile.id is None: - raise RuntimeError("Persisted Embedding Profile is missing its ID") - return profile.id - - @staticmethod - def _require_non_zero_vector(vector) -> None: - if not any(value != 0 for value in vector): - raise InvalidSemanticVectorError("Semantic vectors must be non-zero") - - @staticmethod - def _score(distance: float) -> float: - score = 1.0 - distance - if not math.isfinite(score): - raise InvalidSemanticVectorError("Cosine score must be finite") - return min(1.0, max(-1.0, score)) +if TYPE_CHECKING: + from inkcre_core.business.semantic_retrieval.main import * diff --git a/app/business/sink/agent_query.py b/app/business/sink/agent_query.py index 82e2761c..31b2ac46 100644 --- a/app/business/sink/agent_query.py +++ b/app/business/sink/agent_query.py @@ -8,20 +8,20 @@ import pydantic from starlette.routing import BaseRoute -from app.business.agent import AgentManager -from app.business.job import JobHandler, JobManager +from inkcre_core.business.agent import AgentManager +from inkcre_core.business.job import JobHandler, JobManager from app.middleware import require_peer_jwt -from app.persistence.job.uow import JobUnitOfWork -from app.schemas.ai import ( +from inkcre_core.persistence.job.uow import JobUnitOfWork +from inkcre_core.schemas.ai import ( AssistantMessage, JSONValue, TextContentPart, ToolResultMessage, UserMessage, ) -from app.schemas.job import JobModel -from app.schemas.sink import SinkModel -from libs.obsrv.telemetry import emit_event +from inkcre_core.schemas.job import JobModel +from inkcre_core.schemas.sink import SinkModel +from inkcre_core.libs.obsrv.telemetry import emit_event from .base import SinkBase from .errors import SinkNotFoundError, SinkStateConflictError diff --git a/app/business/sink/base.py b/app/business/sink/base.py index ffc80ff8..845abdca 100644 --- a/app/business/sink/base.py +++ b/app/business/sink/base.py @@ -5,7 +5,7 @@ import fastapi import pydantic -from app.schemas.sink import SinkModel, SinkTypeID +from inkcre_core.schemas.sink import SinkModel, SinkTypeID ConfigT = typing.TypeVar("ConfigT", bound=pydantic.BaseModel) diff --git a/app/business/sink/main.py b/app/business/sink/main.py index 80b51e5f..ab1300ee 100644 --- a/app/business/sink/main.py +++ b/app/business/sink/main.py @@ -8,9 +8,9 @@ import fastapi -from app.persistence.sink.uow import sink_uow -from app.schemas.peer import PeerRef -from app.schemas.sink import SinkID, SinkModel, SinkTypeID, SinkTypeModel +from inkcre_core.persistence.sink.uow import sink_uow +from inkcre_core.schemas.peer import PeerRef +from inkcre_core.schemas.sink import SinkID, SinkModel, SinkTypeID, SinkTypeModel from .errors import ( DuplicateSinkRegistrationError, diff --git a/app/business/sink/mcp.py b/app/business/sink/mcp.py index 2326ef6e..6cdfbe46 100644 --- a/app/business/sink/mcp.py +++ b/app/business/sink/mcp.py @@ -21,19 +21,19 @@ import pydantic from starlette.routing import Mount -from app.business.graph_navigation_retrieval import GraphNavigationRetrievalManager -from app.business.info_base.services import BlockService, get_entity_records -from app.business.info_base.resolver import ( +from inkcre_core.business.graph_navigation_retrieval import GraphNavigationRetrievalManager +from inkcre_core.business.info_base.services import BlockService, get_entity_records +from inkcre_core.business.info_base.resolver import ( ResolverManager, UnknownResolverError, UnsupportedResolverCapability, ) -from app.business.lexical_retrieval import LexicalRetrievalManager -from app.business.semantic_retrieval import SemanticRetrievalManager -from app.schemas.graph_navigation_retrieval import GraphDirection, PathFound -from app.schemas.semantic_retrieval import VectorRetrievalOptions -from app.schemas.sink import SinkModel -from app.schemas.info_base.block import BlockModel +from inkcre_core.business.lexical_retrieval import LexicalRetrievalManager +from inkcre_core.business.semantic_retrieval import SemanticRetrievalManager +from inkcre_core.schemas.graph_navigation_retrieval import GraphDirection, PathFound +from inkcre_core.schemas.semantic_retrieval import VectorRetrievalOptions +from inkcre_core.schemas.sink import SinkModel +from inkcre_core.schemas.info_base.block import BlockModel from .base import SinkBase from .contracts import ContentMode, RecallMode, ResolverMethodCall diff --git a/app/business/sink/projection.py b/app/business/sink/projection.py index e8ca7eee..b4cda815 100644 --- a/app/business/sink/projection.py +++ b/app/business/sink/projection.py @@ -11,8 +11,8 @@ import pydantic -from app.business.info_base.resolver import ResolverManager -from app.schemas.info_base.block import BlockModel +from inkcre_core.business.info_base.resolver import ResolverManager +from inkcre_core.schemas.info_base.block import BlockModel from .contracts import ContentMode diff --git a/app/business/source/__init__.py b/app/business/source/__init__.py index 20b0d0c3..d3ee4e67 100644 --- a/app/business/source/__init__.py +++ b/app/business/source/__init__.py @@ -1,24 +1,3 @@ -from .main import SourceBase, SourceManager -from .config import ( - SOURCE_CONFIG_KEY, - SOURCE_CONFIG_SCHEMA_ID, - SourceDeploymentConfig, -) -from .job import ( - SOURCE_BACKFILL_JOB_TYPE, - SOURCE_COLLECT_JOB_TYPE, - SourceBackfillJobHandler, - SourceCollectJobHandler, -) +"""Legacy import; use inkcre_core for new consumers.""" -__all__ = [ - "SourceBase", - "SourceManager", - "SOURCE_COLLECT_JOB_TYPE", - "SOURCE_BACKFILL_JOB_TYPE", - "SourceCollectJobHandler", - "SourceBackfillJobHandler", - "SOURCE_CONFIG_KEY", - "SOURCE_CONFIG_SCHEMA_ID", - "SourceDeploymentConfig", -] +from inkcre_core.business.source import * diff --git a/app/business/source/config.py b/app/business/source/config.py index 1ed27bd5..4915c28d 100644 --- a/app/business/source/config.py +++ b/app/business/source/config.py @@ -1,46 +1,10 @@ -"""Deployment-wide policy used by Source-owned content materialization.""" +"""Legacy import; use inkcre_core for new consumers.""" -import typing +from typing import TYPE_CHECKING +from importlib import import_module +import sys -import pydantic +sys.modules[__name__] = import_module("inkcre_core.business.source.config") -from app.business.deployment_config import DeploymentConfigManager -from app.business.info_base.storage import StorageManager, WritableStorage -from app.schemas.info_base.storage import StorageID -from app.schemas.source import SourceModel -from app.persistence.source.uow import SourceUnitOfWork - - -SOURCE_CONFIG_KEY = "core.source" -SOURCE_CONFIG_SCHEMA_ID = "core.source.config.v1" -POSTGRESQL_BINARY_STORAGE_ID = -4 - - -class SourceDeploymentConfig(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid") - - default_storage: StorageID = POSTGRESQL_BINARY_STORAGE_ID - - -DeploymentConfigManager.register_schema( - SOURCE_CONFIG_SCHEMA_ID, SourceDeploymentConfig, keys=(SOURCE_CONFIG_KEY,) -) - - -async def resolve_writable_storage_async( - source: SourceModel, uow: SourceUnitOfWork -) -> WritableStorage: - """Resolve storage using the caller's Source/graph/config transaction.""" - storage_id = source.storage - if storage_id is None: - record = await uow.configuration.get(SOURCE_CONFIG_KEY) - persisted = None if record is None else DeploymentConfigManager._restore_record(record) - storage_id = ( - POSTGRESQL_BINARY_STORAGE_ID - if persisted is None - else typing.cast(SourceDeploymentConfig, persisted).default_storage - ) - storage = StorageManager.from_record(await uow.graph.storage.get(storage_id)) - if not isinstance(storage, WritableStorage): - raise ValueError(f"Storage {storage_id} is not writable") - return storage +if TYPE_CHECKING: + from inkcre_core.business.source.config import * diff --git a/app/business/source/job.py b/app/business/source/job.py index 3e80fc84..b84fd5b7 100644 --- a/app/business/source/job.py +++ b/app/business/source/job.py @@ -1,90 +1,10 @@ -"""Exact Source command handlers hosted by the global Job runtime.""" +"""Legacy import; use inkcre_core for new consumers.""" -from app.business.job import JobHandler -from app.persistence.job.uow import JobUnitOfWork -from app.schemas.job import JobModel -from app.schemas.source import ( - SourceBackfillParameters, - SourceCollectParameters, -) +from typing import TYPE_CHECKING +from importlib import import_module +import sys -from app.validation import input_path +sys.modules[__name__] = import_module("inkcre_core.business.source.job") -from .main import SourceManager, SourceNotFoundError - - -SOURCE_COLLECT_JOB_TYPE = "core.source.collect.v1" -SOURCE_BACKFILL_JOB_TYPE = "core.source.backfill.v1" - - -async def _source_type(parameters: SourceCollectParameters) -> str | None: - source = await SourceManager.get(parameters.source) - return None if source is None else source.type - - -class SourceCollectJobHandler( - JobHandler[SourceCollectParameters], - job_type=SOURCE_COLLECT_JOB_TYPE, - description="Run one ordinary collection command for a configured Source.", - parameters_model=SourceCollectParameters, - default_timeout_seconds=300, -): - @classmethod - async def normalize_parameters(cls, parameters: dict, uow: JobUnitOfWork) -> dict: - normalized = await super().normalize_parameters(parameters, uow) - source = await uow.sources.get(normalized["source"]) - if source is None: - raise SourceNotFoundError(f"Source {normalized['source']} does not exist") - with input_path("config"): - normalized["config"] = SourceManager.normalize_config( - source.type, - normalized["config"], - await uow.sources.get_type(source.type), - command="collect", - ) - return normalized - - @classmethod - async def can_handle(cls, parameters: SourceCollectParameters) -> bool: - source_type = await _source_type(parameters) - return source_type is not None and SourceManager.has_source_type(source_type) - - @classmethod - async def handle(cls, job: JobModel, parameters: SourceCollectParameters) -> None: - source = await SourceManager.get_source_ins(parameters.source) - config = source.validate_collect_config(parameters.config) - await source.collect(job, config) - - -class SourceBackfillJobHandler( - JobHandler[SourceBackfillParameters], - job_type=SOURCE_BACKFILL_JOB_TYPE, - description="Run one exact historical collection command for a configured Source.", - parameters_model=SourceBackfillParameters, - default_timeout_seconds=1800, -): - @classmethod - async def normalize_parameters(cls, parameters: dict, uow: JobUnitOfWork) -> dict: - normalized = await super().normalize_parameters(parameters, uow) - source = await uow.sources.get(normalized["source"]) - if source is None: - raise SourceNotFoundError(f"Source {normalized['source']} does not exist") - with input_path("config"): - normalized["config"] = SourceManager.normalize_config( - source.type, - normalized["config"], - await uow.sources.get_type(source.type), - command="backfill", - ) - return normalized - - @classmethod - async def can_handle(cls, parameters: SourceBackfillParameters) -> bool: - source_type = await _source_type(parameters) - return source_type is not None and SourceManager.supports_backfill(source_type) - - @classmethod - async def handle(cls, job: JobModel, parameters: SourceBackfillParameters) -> None: - source = await SourceManager.get_source_ins(parameters.source) - config = source.validate_backfill_config(parameters.config) - await source.backfill(job, config) +if TYPE_CHECKING: + from inkcre_core.business.source.job import * diff --git a/app/business/source/main.py b/app/business/source/main.py index 2ceb8fb2..9a31a59a 100644 --- a/app/business/source/main.py +++ b/app/business/source/main.py @@ -1,314 +1,10 @@ -from __future__ import annotations +"""Legacy import; use inkcre_core for new consumers.""" -import abc -import jsonschema # pyrefly: ignore[untyped-import] -import pydantic -import typing -from typing import Optional as Opt +from typing import TYPE_CHECKING +from importlib import import_module +import sys -from app.persistence.source.uow import SourceUnitOfWork, source_uow -from app.schemas.info_base.block import BlockForm, BlockModel -from app.schemas.job import JobModel -from app.schemas.source import SourceModel, SourceID, SourceTypesModel, SourceUpdateForm -from app.validation import input_path -from .resolver import SOURCE_RESOLVER_ID, SourceContent +sys.modules[__name__] = import_module("inkcre_core.business.source.main") -ConfigTV = typing.TypeVar("ConfigTV", bound=pydantic.BaseModel) - - -class SourceNotFoundError(LookupError): - """A Source instance or persisted type does not exist.""" - - -class UnsupportedSourceCommandError(ValueError): - """The catalog explicitly does not support the requested collection mode.""" - - -class EmptySourceCommandConfig(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid") - - -class SourceBase(abc.ABC, typing.Generic[ConfigTV]): - """InkCre Source Base class. - - Configuration is loaded from the database. Instantiate a source when it is - used instead of caching configuration separately. - """ - - __configschema__: dict - """Source configuration JSON schema""" - __configcls__: type[ConfigTV] - __collectconfigcls__: type[pydantic.BaseModel] - __backfillconfigcls__: type[pydantic.BaseModel] | None - - def __init_subclass__( - cls, - config_cls: type[ConfigTV], - collect_config_cls: type[pydantic.BaseModel] = EmptySourceCommandConfig, - backfill_config_cls: type[pydantic.BaseModel] | None = None, - **kwargs, - ) -> None: - # ConfigTV is bound by the concrete source subclass. - cls.__configcls__ = config_cls # pyrefly: ignore[no-access] - cls.__configschema__ = config_cls.model_json_schema() - cls.__collectconfigcls__ = collect_config_cls # pyrefly: ignore[no-access] - cls.__backfillconfigcls__ = backfill_config_cls # pyrefly: ignore[no-access] - SourceManager.add_source_type(cls) - return super().__init_subclass__(**kwargs) - - def __init__(self, _id: SourceID) -> None: - self._id = _id - - @abc.abstractmethod - async def collect(self, job: JobModel, config: pydantic.BaseModel) -> None: - """Collect new data from the source. - - :param job: The collect job containing config and state. - - Notes: - - Should not surpress exceptions, raise it. - """ - - async def backfill(self, job: JobModel, config: pydantic.BaseModel) -> None: - del job, config - raise NotImplementedError(f"{self.__class__.__name__} does not support backfill") - - async def record(self, data: typing.Any) -> None: - """Record data passively (e.g., from webhook). - - :param data: The data to record from external source (e.g., webhook payload). - - Notes: - - Should not suppress exceptions, raise it. - - Used for passive collection methods like webhooks. - """ - raise NotImplementedError(f"{self.__class__.__name__} does not support passive record") - - async def get_config(self) -> ConfigTV: - """Read and validate current Source configuration in a short scope.""" - async with source_uow() as uow: - source = await uow.sources.get(self._id) - if source is None: - raise SourceNotFoundError(f"Source {self._id} does not exist") - return self.__configcls__.model_validate(source.config) - - def validate_collect_config(self, config: dict) -> pydantic.BaseModel: - return self.__collectconfigcls__.model_validate(config) - - def validate_backfill_config(self, config: dict) -> pydantic.BaseModel: - if self.__backfillconfigcls__ is None: - raise NotImplementedError(f"{self.__class__.__name__} does not support backfill") - return self.__backfillconfigcls__.model_validate(config) - - async def get_state(self) -> dict: - async with source_uow() as uow: - source = await uow.sources.get(self._id) - if source is None: - raise SourceNotFoundError(f"Source {self._id} does not exist") - return source.state or {} - - async def set_state(self, state: dict) -> None: - async with source_uow() as uow: - source = await uow.sources.get(self._id, lock=True) - if source is None: - raise SourceNotFoundError(f"Source {self._id} does not exist") - source.state = state - await uow.sources.save(source) - - -class SourceManager: - """ - - - Run collect method of all configured sources - - Add, remove and configure source instances - - Add, remove sources - """ - - SOURCES: dict[SourceID, SourceBase] = {} - _SOURCE_CLASSES: dict[str, type[SourceBase]] = {} - - @classmethod - def add_source_type(cls, source_cls: type[SourceBase]) -> None: - """Register a source type in memory without external side effects.""" - source_type = source_cls.__module__ + "." + source_cls.__qualname__ - cls._SOURCE_CLASSES[source_type] = source_cls - - @classmethod - async def sync_source_types_async( - cls, source_classes: dict[str, type[SourceBase]] | None = None - ) -> None: - registered = cls._SOURCE_CLASSES if source_classes is None else source_classes - rows = [ - dict( - id=source_type, - description=source_cls.__doc__ or "No description.", - config_schema=source_cls.__configschema__, - collect_config_schema=source_cls.__collectconfigcls__.model_json_schema(), - backfill_config_schema=( - None - if source_cls.__backfillconfigcls__ is None - else source_cls.__backfillconfigcls__.model_json_schema() - ), - ) - for source_type, source_cls in registered.items() - ] - async with source_uow() as uow: - await uow.sources.sync_types(rows) - - @classmethod - def has_source_type(cls, source_type: str) -> bool: - return source_type in cls._SOURCE_CLASSES - - @classmethod - def supports_backfill(cls, source_type: str) -> bool: - source_cls = cls._SOURCE_CLASSES.get(source_type) - return source_cls is not None and source_cls.__backfillconfigcls__ is not None - - @classmethod - async def get_source_ins(cls, source_id: SourceID) -> SourceBase: - ins = cls.SOURCES.get(source_id) - if ins is not None: - return ins - source = await cls.get(source_id) - if source is None: - raise SourceNotFoundError(f"Source {source_id} does not exist") - source_class = cls._SOURCE_CLASSES.get(source.type) - if source_class is None: - raise ValueError(f"Source class {source.type} not registered.") - ins = source_class(_id=source_id) - cls.SOURCES[source_id] = ins - return ins - - @classmethod - async def create( - cls, - type_: str, - nickname: Opt[str] = None, - config: dict | None = None, - storage: int | None = None, - ) -> SourceModel: - async with source_uow() as uow: - with input_path("config"): - normalized = cls.normalize_config( - type_, config or {}, await uow.sources.get_type(type_) - ) - source = SourceModel( - type=type_, nickname=nickname, config=normalized, storage=storage - ) - await uow.sources.save(source) - return source - - @classmethod - def normalize_config( - cls, - type_: str, - config: dict, - catalog: SourceTypesModel | None, - *, - command: typing.Literal["collect", "backfill"] | None = None, - ) -> dict: - """Accept catalog-only input without requiring local execution capability.""" - if catalog is None: - raise SourceNotFoundError(f"Source type {type_!r} does not exist") - source_class = cls._SOURCE_CLASSES.get(type_) - if command == "backfill": - schema = catalog.backfill_config_schema - model = source_class.__backfillconfigcls__ if source_class else None - elif command == "collect": - schema = catalog.collect_config_schema - model = source_class.__collectconfigcls__ if source_class else None - else: - schema = catalog.config_schema - model = source_class.__configcls__ if source_class else None # pyrefly: ignore[missing-attribute] - if schema is None: - raise UnsupportedSourceCommandError( - f"Source type {type_!r} does not support {command}" - ) - if model is not None: - return model.model_validate(config).model_dump(mode="json") - jsonschema.Draft202012Validator(schema).validate(config) - return config - - @classmethod - async def get(cls, source_id: SourceID) -> SourceModel | None: - async with source_uow() as uow: - return await uow.sources.get(source_id) - - @classmethod - async def get_type(cls, type_: str) -> SourceTypesModel | None: - async with source_uow() as uow: - return await uow.sources.get_type(type_) - - @classmethod - async def list_sources( - cls, *, limit: int | None = None, cursor: SourceID | None = None - ) -> tuple[list[SourceModel], SourceID | None]: - async with source_uow() as uow: - return await uow.sources.list_sources(limit=limit, cursor=cursor) - - @classmethod - async def list_types( - cls, *, limit: int | None = None, cursor: str | None = None - ) -> tuple[list[SourceTypesModel], str | None]: - async with source_uow() as uow: - return await uow.sources.list_types(limit=limit, cursor=cursor) - - @classmethod - async def update(cls, source_id: SourceID, form: SourceUpdateForm) -> SourceModel: - async with source_uow() as uow: - source = await uow.sources.get(source_id, lock=True) - if source is None: - raise SourceNotFoundError(f"Source {source_id} does not exist") - changes = form.model_dump(exclude_unset=True) - if "config" in changes: - with input_path("config"): - changes["config"] = cls.normalize_config( - source.type, changes["config"], await uow.sources.get_type(source.type) - ) - for field, value in changes.items(): - setattr(source, field, value) - await uow.sources.save(source) - if "nickname" in changes and source.block is not None: - await cls.ensure_block_async(source, uow) - return source - - @classmethod - async def delete(cls, source_id: SourceID) -> bool: - """Delete configuration only; collected graph and scheduled commands remain.""" - async with source_uow() as uow: - source = await uow.sources.get(source_id) - if source is None: - return False - await uow.sources.delete(source) - cls.SOURCES.pop(source_id, None) - return True - - @classmethod - async def ensure_block_async( - cls, source: SourceModel, uow: SourceUnitOfWork - ) -> BlockModel: - """Create/reuse a Source anchor in the caller's graph/state transaction.""" - if source.id is None: - raise ValueError("Source must be persisted before creating its anchor") - locked = await uow.sources.get(source.id, lock=True) - if locked is None: - raise SourceNotFoundError(f"Source {source.id} does not exist") - content = SourceContent( - id=source.id, type=locked.type, nickname=locked.nickname - ).model_dump_json() - if locked.block is None: - block = await uow.graph.blocks.create( - BlockForm(resolver=SOURCE_RESOLVER_ID, content=content) - ) - locked.block = block.id - await uow.sources.save(locked) - return block - block = await uow.graph.blocks.get(locked.block) - if block is None: - raise RuntimeError("Source anchor reference does not resolve") - if block.resolver != SOURCE_RESOLVER_ID or block.content != content: - block.resolver = SOURCE_RESOLVER_ID - block.storage = None - block.content = content - await uow.graph.blocks.save(block) - return block +if TYPE_CHECKING: + from inkcre_core.business.source.main import * diff --git a/app/business/source/resolver.py b/app/business/source/resolver.py index 78b70056..157534fc 100644 --- a/app/business/source/resolver.py +++ b/app/business/source/resolver.py @@ -1,47 +1,10 @@ -"""Use-time projection for lazy Source anchor Blocks.""" +"""Legacy import; use inkcre_core for new consumers.""" -import pydantic +from typing import TYPE_CHECKING +from importlib import import_module +import sys -from app.business.info_base.resolver.label import format_label -from app.business.info_base.resolver.main import Resolver, TextProjectionContext +sys.modules[__name__] = import_module("inkcre_core.business.source.resolver") - -SOURCE_RESOLVER_ID = "core.source.v1" - - -class SourceContent(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid") - - id: int - type: str - nickname: str | None = None - - -class SourceResolver(Resolver[SourceContent, str], rso_type=SOURCE_RESOLVER_ID): - async def _get_solved_content( - self, - *, - refresh: bool = False, - materialize_missing: bool = True, - ) -> SourceContent: - del materialize_missing - return SourceContent.model_validate_json(await self.get_raw_content(refresh=refresh)) - - async def get_text( - self, - *, - context: TextProjectionContext = "default", - refresh: bool = False, - materialize_missing: bool = True, - ) -> str: - del context - source = await self.get_solved_content( - refresh=refresh, - materialize_missing=materialize_missing, - ) - name = f" named {source.nickname}" if source.nickname else "" - return f"Source {source.id}{name} uses {source.type}." - - async def get_label(self, *, refresh: bool = False) -> str: - source = await self.get_solved_content(refresh=refresh, materialize_missing=False) - return format_label("source", source.nickname or source.type) +if TYPE_CHECKING: + from inkcre_core.business.source.resolver import * diff --git a/app/business/source/webpage.py b/app/business/source/webpage.py index 74814835..1479c42e 100644 --- a/app/business/source/webpage.py +++ b/app/business/source/webpage.py @@ -1 +1,10 @@ -"""Webpage Source""" +"""Legacy import; use inkcre_core for new consumers.""" + +from typing import TYPE_CHECKING +from importlib import import_module +import sys + +sys.modules[__name__] = import_module("inkcre_core.business.source.webpage") + +if TYPE_CHECKING: + from inkcre_core.business.source.webpage import * diff --git a/app/configuration.py b/app/configuration.py index c09431af..acac60aa 100644 --- a/app/configuration.py +++ b/app/configuration.py @@ -1,59 +1,10 @@ -"""Persistence-neutral mechanics for model-owned configuration values.""" +"""Legacy import; use inkcre_core for new consumers.""" -from collections.abc import Mapping -import typing +from typing import TYPE_CHECKING +from importlib import import_module +import sys -import pydantic +sys.modules[__name__] = import_module("inkcre_core.configuration") - -ConfigModelT = typing.TypeVar("ConfigModelT", bound=pydantic.BaseModel) - - -class ConfigContract(typing.Generic[ConfigModelT]): - """Apply one Pydantic model as a complete configuration contract. - - This abstraction deliberately has no persistence, registry, key, or live-update - behavior. Those consequences remain with the domain that owns the config. - """ - - def __init__(self, model: type[ConfigModelT]) -> None: - self.model = model - - def validate( - self, - value: ConfigModelT | Mapping[str, typing.Any], - ) -> ConfigModelT: - """Validate new input; pass an already constructed target model through. - - Passing a model back through Pydantic can rerun its model validators. - Typed callers have already crossed that boundary; persistence consumers may - still use one native model construction to restore nested Python types. - """ - if isinstance(value, self.model): - return value - return self.model.model_validate(value) - - def normalize( - self, - value: ConfigModelT | Mapping[str, typing.Any], - ) -> dict[str, typing.Any]: - """Return the complete value in JSON-compatible canonical form.""" - return self.validate(value).model_dump(mode="json") - - def prepare_patch( - self, - current: ConfigModelT | Mapping[str, typing.Any], - patch: Mapping[str, typing.Any], - ) -> ConfigModelT: - """Shallow-merge a patch and validate the resulting complete value.""" - candidate = ( - current.model_dump(mode="json") - if isinstance(current, pydantic.BaseModel) - else dict(current) - ) - candidate.update(patch) - return self.validate(candidate) - - def json_schema(self) -> dict[str, typing.Any]: - """Project the model's JSON Schema for UI or protocol discovery.""" - return self.model.model_json_schema() +if TYPE_CHECKING: + from inkcre_core.configuration import * diff --git a/app/core.py b/app/core.py new file mode 100644 index 00000000..03abb695 --- /dev/null +++ b/app/core.py @@ -0,0 +1,39 @@ +"""Application-owned composition of the embeddable SDK.""" + +from starlette.types import ASGIApp, Receive, Scope, Send + +from inkcre_core import Core, CoreOptions +from app.settings import settings +from app.version import APPLICATION_VERSION + + +def create_core() -> Core: + """Compose one service-owned SDK instance from application settings.""" + return Core( + settings.database_url, + pool_pre_ping=settings.database_scale_0, + options=CoreOptions( + peer_id=settings.peer_id, + peer_name=settings.peer_name, + application_version=APPLICATION_VERSION, + jwt_secret=settings.jwt_secret, + jwt_default_lifetime_seconds=settings.jwt_default_lifetime_seconds, + peer_http_timeout_seconds=settings.peer_http_timeout_seconds, + agent_debug=settings.obsrv.agent_debug, + ), + ) + + +CORE = create_core() + + +class CoreMiddleware: + """Bind the application's SDK instance for requests, lifespan and child work.""" + + def __init__(self, app: ASGIApp, core: Core): + self.app = app + self.core = core + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + with self.core.bind(): + await self.app(scope, receive, send) diff --git a/app/database_contract/__init__.py b/app/database_contract/__init__.py index 8943ac4e..ed99b743 100644 --- a/app/database_contract/__init__.py +++ b/app/database_contract/__init__.py @@ -1,29 +1,3 @@ -"""Executable peer database runtime contract.""" +"""Legacy import; use inkcre_core for new consumers.""" -from .constants import ( - AUTHENTICATED_ROLE, - AUTHENTICATOR_ROLE, - CONTRACT_REVISION, - CORE_RUNTIME_ROLE, - DEVELOPMENT_PEER_ID, - INTERNAL_SCHEMA, - JWT_ALGORITHM, - JWT_AUDIENCE, - JWT_ISSUER, - JWT_MAX_LIFETIME_SECONDS, - PROTOCOL_SCHEMA, -) - -__all__ = [ - "AUTHENTICATED_ROLE", - "AUTHENTICATOR_ROLE", - "CONTRACT_REVISION", - "CORE_RUNTIME_ROLE", - "DEVELOPMENT_PEER_ID", - "INTERNAL_SCHEMA", - "JWT_ALGORITHM", - "JWT_AUDIENCE", - "JWT_ISSUER", - "JWT_MAX_LIFETIME_SECONDS", - "PROTOCOL_SCHEMA", -] +from inkcre_core.database_contract import * diff --git a/app/database_contract/constants.py b/app/database_contract/constants.py index c5dd790f..93355ec8 100644 --- a/app/database_contract/constants.py +++ b/app/database_contract/constants.py @@ -1,57 +1,10 @@ -"""Stable names and values shared by migrations, runtimes, and consumers.""" +"""Legacy import; use inkcre_core for new consumers.""" -import uuid +from typing import TYPE_CHECKING +from importlib import import_module +import sys +sys.modules[__name__] = import_module("inkcre_core.database_contract.constants") -CONTRACT_FORMAT = 1 -CONTRACT_REVISION = "extension-distribution-open-v1" - -PROTOCOL_SCHEMA = "inkcre" -INTERNAL_SCHEMA = "inkcre_internal" - -AUTHENTICATED_ROLE = "authenticated" -AUTHENTICATOR_ROLE = "authenticator" -ANONYMOUS_ROLE = "anonymous" -CORE_RUNTIME_ROLE = "inkcre_core" - -JWT_ALGORITHM = "HS256" -JWT_ROLE = AUTHENTICATED_ROLE -JWT_ISSUER = "inkcre-peer" -JWT_AUDIENCE = "inkcre-api" -# Frozen input for admitted historical migrations; live admission has no lifetime cap. -JWT_MAX_LIFETIME_SECONDS = 24 * 60 * 60 -JWT_MINIMUM_SECRET_BYTES = 32 - -DEVELOPMENT_PEER_ID = uuid.UUID("00000000-0000-4000-8000-000000000001") -DEVELOPMENT_PEER_NAME = "client-web-development" - -DATABASE_ENVIRONMENTS = frozenset({"runtime", "development", "preview", "production"}) -RESET_CONFIRMATION = "reset-development-data" - -APPLICATION_TABLES = ( - "agents", - "ai_dialects", - "ai_models", - "ai_providers", - "block_embeddings", - "block_lexical_records", - "blocks", - "peers", - "configs", - "extensions", - "extension_peer_participations", - "embedding_profiles", - "crons", - "jobs", - "job_types", - "logs", - "relation_embeddings", - "relations", - "sink_types", - "sinks", - "sources", - "sources_types", - "storage_types", - "storage_blobs", - "storages", -) +if TYPE_CHECKING: + from inkcre_core.database_contract.constants import * diff --git a/app/database_contract/profile.py b/app/database_contract/profile.py index bfb56505..f1990cc2 100644 --- a/app/database_contract/profile.py +++ b/app/database_contract/profile.py @@ -1,386 +1,10 @@ -"""Checked-in built-in catalog profile. +"""Legacy import; use inkcre_core for new consumers.""" -The profile is data only. Reconciliation can consume it without importing FastAPI, -application settings, extension runtimes, or the scheduler. -""" +from typing import TYPE_CHECKING +from importlib import import_module +import sys -from dataclasses import dataclass -from typing import Any +sys.modules[__name__] = import_module("inkcre_core.database_contract.profile") - -JsonObject = dict[str, Any] - - -@dataclass(frozen=True) -class TypeProfile: - id: str - description: str - config_schema: JsonObject - - -@dataclass(frozen=True) -class StorageTypeProfile: - id: str - description: str - config_schema: JsonObject - writable: bool - - -@dataclass(frozen=True) -class JobTypeProfile: - id: str - description: str - parameters_schema: JsonObject - default_timeout_seconds: int - - -@dataclass(frozen=True) -class StorageProfile: - id: int - type: str - nickname: str - config: JsonObject - - -def _object_schema( - title: str, - description: str, - properties: JsonObject, -) -> JsonObject: - return { - "description": description, - "properties": properties, - "title": title, - "type": "object", - } - - -def _positive_integer(default: int) -> JsonObject: - return {"default": default, "exclusiveMinimum": 0, "type": "integer"} - - -def _boolean(default: bool) -> JsonObject: - return {"default": default, "type": "boolean"} - - -HTTP_STORAGE_SCHEMA = _object_schema( - "HTTPStorageConfig", - "Configuration for HTTP storage.", - { - "follow_redirects": _boolean(True), - "max_response_bytes": _positive_integer(64 * 1024 * 1024), - "timeout": _positive_integer(30), - }, -) - -OPENAI_COMPATIBLE_DIALECT_SCHEMA = { - "additionalProperties": False, - "description": "Connection values for one OpenAI-compatible provider instance.", - "properties": { - "api_key": { - "minLength": 1, - "title": "Api Key", - "type": "string", - }, - "base_url": { - "anyOf": [ - {"minLength": 1, "type": "string"}, - {"type": "null"}, - ], - "default": None, - "title": "Base Url", - }, - }, - "required": ["api_key"], - "title": "OpenAICompatibleConfig", - "type": "object", -} - -BUILTIN_AI_DIALECTS = ( - TypeProfile( - "core.openai-compatible.v1", - "OpenAI-compatible embedding and chat protocol.", - OPENAI_COMPATIBLE_DIALECT_SCHEMA, - ), - TypeProfile( - "core.alibaba-model-studio.v1", - "Alibaba Model Studio OpenAI-compatible multimodal chat protocol.", - OPENAI_COMPATIBLE_DIALECT_SCHEMA, - ), -) - -BUILTIN_STORAGE_TYPES = ( - StorageTypeProfile( - "http", - "HTTP storage for bounded retrieval of opaque response bytes.", - HTTP_STORAGE_SCHEMA, - False, - ), -) - -BUILTIN_STORAGE_TYPES += ( - StorageTypeProfile( - "postgresql_binary", - "PostgreSQL storage for deployment-owned binary content.", - _object_schema( - "PostgreSQLBinaryStorageConfig", - "Configuration for PostgreSQL binary storage.", - {}, - ), - True, - ), -) - - -SOURCE_COLLECT_PARAMETERS_SCHEMA = { - "additionalProperties": False, - "description": "Parameters of the exact ordinary Source Job type.", - "properties": { - "config": { - "additionalProperties": True, - "title": "Config", - "type": "object", - }, - "source": {"title": "Source", "type": "integer"}, - }, - "required": ["source"], - "title": "SourceCollectParameters", - "type": "object", -} - -SOURCE_BACKFILL_PARAMETERS_SCHEMA = { - **SOURCE_COLLECT_PARAMETERS_SCHEMA, - "description": "Parameters of the exact historical Source Job type.", - "title": "SourceBackfillParameters", -} - -LEXICAL_MAINTENANCE_PARAMETERS_SCHEMA = { - "$defs": { - "LexicalMaintenanceOptions": { - "additionalProperties": False, - "properties": { - "diagnostic_limit": { - "default": 20, - "minimum": 0, - "title": "Diagnostic Limit", - "type": "integer", - }, - "max_records": { - "default": 100, - "minimum": 1, - "title": "Max Records", - "type": "integer", - }, - "scan_page_size": { - "default": 100, - "minimum": 1, - "title": "Scan Page Size", - "type": "integer", - }, - }, - "title": "LexicalMaintenanceOptions", - "type": "object", - } - }, - "additionalProperties": False, - "properties": { - "options": {"$ref": "#/$defs/LexicalMaintenanceOptions"}, - }, - "title": "LexicalMaintenanceJobParameters", - "type": "object", -} - -SEMANTIC_MAINTENANCE_PARAMETERS_SCHEMA = { - "$defs": { - "EmbeddingMaintenanceOptions": { - "additionalProperties": False, - "description": "Peer-local bounds for one resumable maintenance invocation.", - "properties": { - "batch_size": { - "default": 20, - "minimum": 1, - "title": "Batch Size", - "type": "integer", - }, - "diagnostic_limit": { - "default": 20, - "minimum": 0, - "title": "Diagnostic Limit", - "type": "integer", - }, - "max_embeddings": { - "default": 100, - "minimum": 1, - "title": "Max Embeddings", - "type": "integer", - }, - "scan_page_size": { - "default": 100, - "minimum": 1, - "title": "Scan Page Size", - "type": "integer", - }, - }, - "title": "EmbeddingMaintenanceOptions", - "type": "object", - } - }, - "additionalProperties": False, - "properties": { - "options": {"$ref": "#/$defs/EmbeddingMaintenanceOptions"}, - "profile": { - "anyOf": [{"type": "integer"}, {"type": "null"}], - "default": None, - "title": "Profile", - }, - }, - "title": "EmbeddingMaintenanceJobParameters", - "type": "object", -} - -MEDIA_INTERPRETATION_PARAMETERS_SCHEMA = { - "additionalProperties": False, - "properties": {}, - "title": "MediaInterpretationJobParameters", - "type": "object", -} - -AUTOMATIC_ORGANIZATION_PARAMETERS_SCHEMA = { - "additionalProperties": False, - "properties": { - "max_seeds": { - "default": 10, - "maximum": 100, - "minimum": 3, - "title": "Max Seeds", - "type": "integer", - } - }, - "title": "AutomaticOrganizationJobParameters", - "type": "object", -} - -BUILTIN_JOB_TYPES = ( - JobTypeProfile( - "core.source.collect.v1", - "Run one ordinary collection command for a configured Source.", - SOURCE_COLLECT_PARAMETERS_SCHEMA, - 300, - ), - JobTypeProfile( - "core.source.backfill.v1", - "Run one exact historical collection command for a configured Source.", - SOURCE_BACKFILL_PARAMETERS_SCHEMA, - 1800, - ), - JobTypeProfile( - "core.feature_retrieval.lexical.maintain.v1", - "Maintain missing or stale Block lexical projection records.", - LEXICAL_MAINTENANCE_PARAMETERS_SCHEMA, - 900, - ), - JobTypeProfile( - "core.feature_retrieval.lexical.rebuild.v1", - "Rebuild Block lexical records present before invocation.", - LEXICAL_MAINTENANCE_PARAMETERS_SCHEMA, - 1800, - ), - JobTypeProfile( - "core.semantic_retrieval.maintain.v1", - "Maintain missing or stale semantic embedding records.", - SEMANTIC_MAINTENANCE_PARAMETERS_SCHEMA, - 900, - ), - JobTypeProfile( - "core.semantic_retrieval.rebuild.v1", - "Rebuild semantic embedding records present before invocation.", - SEMANTIC_MAINTENANCE_PARAMETERS_SCHEMA, - 1800, - ), - JobTypeProfile( - "core.organization.media_interpretation.v1", - "Interpret missing image, audio, and video Blocks through configured Agents.", - MEDIA_INTERPRETATION_PARAMETERS_SCHEMA, - 1800, - ), - JobTypeProfile( - "core.organization.rumination.explicit.v1", - "Reconsider one explicitly selected Block through rumination.", - { - "additionalProperties": False, - "description": "Explicit request to reconsider one focal Block.", - "properties": {"block": {"title": "Block", "type": "integer"}}, - "required": ["block"], - "title": "RuminationRequest", - "type": "object", - }, - 1800, - ), - JobTypeProfile( - "core.organization.rumination.automatic.v1", - "Automatically reconsider bounded information seeds through rumination.", - AUTOMATIC_ORGANIZATION_PARAMETERS_SCHEMA, - 1800, - ), - JobTypeProfile( - "core.organization.supersession.automatic.v1", - "Automatically judge bounded scoped-supersession candidates.", - AUTOMATIC_ORGANIZATION_PARAMETERS_SCHEMA, - 1800, - ), - JobTypeProfile( - "core.organization.refinement.automatic.v1", - "Automatically judge bounded non-dominating refinement candidates.", - AUTOMATIC_ORGANIZATION_PARAMETERS_SCHEMA, - 1800, - ), - JobTypeProfile( - "core.organization.evidence-stance.automatic.v1", - "Automatically judge bounded attributable evidence stances.", - AUTOMATIC_ORGANIZATION_PARAMETERS_SCHEMA, - 1800, - ), - JobTypeProfile( - "core.organization.synthesis.automatic.v1", - "Automatically create bounded provenance-preserving syntheses.", - AUTOMATIC_ORGANIZATION_PARAMETERS_SCHEMA, - 1800, - ), - JobTypeProfile( - "core.organization.existing-referent-anchoring.automatic.v1", - "Automatically anchor bounded mentions to existing referents.", - AUTOMATIC_ORGANIZATION_PARAMETERS_SCHEMA, - 1800, - ), - JobTypeProfile( - "core.organization.duplicate-assertion.automatic.v1", - "Automatically judge bounded provenance-aware duplicate assertions.", - AUTOMATIC_ORGANIZATION_PARAMETERS_SCHEMA, - 1800, - ), - JobTypeProfile( - "core.sink.agent-query.v1", - "Answer one information need through an enabled Agent Query Sink.", - { - "additionalProperties": False, - "properties": { - "query": {"minLength": 1, "title": "Query", "type": "string"}, - "sink": {"title": "Sink", "type": "integer"}, - }, - "required": ["sink", "query"], - "title": "AgentQueryJobParameters", - "type": "object", - }, - 300, - ), -) - - -BUILTIN_STORAGES = ( - StorageProfile(-1, "http", "HTTP", {}), - StorageProfile(-4, "postgresql_binary", "PostgreSQL Binary", {}), -) - -BUILTIN_AI_DIALECTS_BY_ID = {item.id: item for item in BUILTIN_AI_DIALECTS} -BUILTIN_STORAGE_TYPES_BY_ID = {item.id: item for item in BUILTIN_STORAGE_TYPES} -BUILTIN_JOB_TYPES_BY_ID = {item.id: item for item in BUILTIN_JOB_TYPES} +if TYPE_CHECKING: + from inkcre_core.database_contract.profile import * diff --git a/app/engine.py b/app/engine.py index 41aefacf..842da442 100644 --- a/app/engine.py +++ b/app/engine.py @@ -1,13 +1,8 @@ -"""The process-owned async engine and session factory; lifespan disposes the pool.""" +"""Legacy import; use inkcre_core for new consumers.""" -from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine +from app.core import CORE -from app.settings import settings - - -ASYNC_DB_ENGINE = create_async_engine( - settings.database_url, pool_pre_ping=settings.database_scale_0 -) -AsyncSessionFactory = async_sessionmaker(ASYNC_DB_ENGINE, expire_on_commit=False) +ASYNC_DB_ENGINE = CORE.engine +AsyncSessionFactory = CORE.sessions __all__ = ["ASYNC_DB_ENGINE", "AsyncSessionFactory"] diff --git a/app/http.py b/app/http.py index 992b0f17..3328f500 100644 --- a/app/http.py +++ b/app/http.py @@ -1,6 +1,6 @@ """Public HTTP address supplied by this Core's deployment configuration.""" -from app.business.peer import PeerManager +from inkcre_core.business.peer import PeerManager async def get_public_http_base_url() -> str | None: diff --git a/app/middleware.py b/app/middleware.py index 7a16662e..62249f99 100644 --- a/app/middleware.py +++ b/app/middleware.py @@ -1,6 +1,5 @@ """Middleware for logging and request tracking.""" -import math import time from typing import Callable import uuid @@ -10,15 +9,10 @@ from starlette.middleware.base import BaseHTTPMiddleware from starlette.types import ASGIApp -from app.database_contract.constants import ( - JWT_ALGORITHM, - JWT_AUDIENCE, - JWT_ISSUER, - JWT_ROLE, -) +from inkcre_core.auth import decode_peer_jwt, create_peer_jwt as sign_peer_jwt from app.settings import settings -from libs.obsrv.log_record import TRACE_ID -from libs.obsrv.main import get_logger +from inkcre_core.libs.obsrv.log_record import TRACE_ID +from inkcre_core.libs.obsrv.main import get_logger SENSITIVE_QUERY_PARAMETERS = frozenset( @@ -34,75 +28,15 @@ def logged_query_params(request: Request) -> str: ) -def decode_peer_jwt( - token: str, - secret: str, - *, - now: float | None = None, -) -> dict: - """Validate the canonical peer JWT contract used by every HTTP surface.""" - claims = jwt.decode( - token, - secret, - algorithms=[JWT_ALGORITHM], - audience=JWT_AUDIENCE, - issuer=JWT_ISSUER, - options={ - "require": ["role", "iss", "aud", "iat", "exp"], - "verify_exp": False, - "verify_iat": False, - }, - ) - issued_at = claims["iat"] - expires_at = claims["exp"] - valid_numeric_dates = ( - isinstance(issued_at, (int, float)) - and not isinstance(issued_at, bool) - and isinstance(expires_at, (int, float)) - and not isinstance(expires_at, bool) - ) - if not valid_numeric_dates: - raise jwt.exceptions.InvalidTokenError("JWT claim contract mismatch") - try: - issued_at_number = float(issued_at) - expires_at_number = float(expires_at) - except OverflowError as error: - raise jwt.exceptions.InvalidTokenError("JWT claim contract mismatch") from error - current_epoch = time.time() if now is None else now - if ( - claims["role"] != JWT_ROLE - or not math.isfinite(issued_at_number) - or not math.isfinite(expires_at_number) - or issued_at_number > current_epoch + 60 - or expires_at_number <= current_epoch - or expires_at_number <= issued_at_number - ): - raise jwt.exceptions.InvalidTokenError("JWT claim contract mismatch") - return claims - - def create_peer_jwt( - secret: str, - *, - now: float | None = None, - lifetime_seconds: int | None = None, + secret: str, *, now: float | None = None, lifetime_seconds: int | None = None ) -> str: - """Sign a Peer JWT using the configured default when no lifetime is supplied.""" - if lifetime_seconds is None: - lifetime_seconds = settings.jwt_default_lifetime_seconds - if lifetime_seconds <= 0: - raise ValueError("Peer JWT lifetime must be positive") - issued_at = int(time.time() if now is None else now) - return jwt.encode( - { - "role": JWT_ROLE, - "iss": JWT_ISSUER, - "aud": JWT_AUDIENCE, - "iat": issued_at, - "exp": issued_at + lifetime_seconds, - }, + return sign_peer_jwt( secret, - algorithm=JWT_ALGORITHM, + now=now, + lifetime_seconds=settings.jwt_default_lifetime_seconds + if lifetime_seconds is None + else lifetime_seconds, ) diff --git a/app/observability.py b/app/observability.py index d3a2fda2..7b31e1ef 100644 --- a/app/observability.py +++ b/app/observability.py @@ -1,23 +1,27 @@ """Runtime projection of optional deployment telemetry and context boundaries.""" import asyncio -from collections.abc import Mapping -from opentelemetry.context import Context -from opentelemetry.trace import Link, SpanKind -from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator -from opentelemetry import trace +from opentelemetry.trace import SpanKind from starlette.types import ASGIApp, Message, Receive, Scope, Send -from app.business.deployment_config import DeploymentConfigManager, DeploymentConfigService +from inkcre_core.business.deployment_config import ( + DeploymentConfigManager, + DeploymentConfigService, +) from app.settings import settings -from app.schemas.observability import CONFIG_KEY, CONFIG_SCHEMA, ObservabilityConfig +from inkcre_core.schemas.observability import CONFIG_KEY, CONFIG_SCHEMA, ObservabilityConfig from app.version import APPLICATION_VERSION -from libs.obsrv.main import get_logger -from libs.obsrv.telemetry import emit_event, is_enabled, operation, start_telemetry +from inkcre_core.libs.obsrv.main import get_logger +from inkcre_core.libs.obsrv.telemetry import is_enabled, operation, start_telemetry -PROPAGATOR = TraceContextTextMapPropagator() +from inkcre_core.libs.obsrv.propagation import ( + extract_context, + inject_context as inject_context, + submission_context as submission_context, + submission_links as submission_links, +) DeploymentConfigManager.register_schema( @@ -54,48 +58,6 @@ async def initialize_telemetry() -> None: get_logger().warning("Telemetry initialization failed (%s)", type(error).__name__) -def inject_context() -> dict[str, str]: - """Inject only standard Trace Context when this runtime explicitly enabled telemetry.""" - carrier: dict[str, str] = {} - if is_enabled(): - PROPAGATOR.inject(carrier) - return carrier - - -def submission_context() -> dict[str, str | None]: - carrier = inject_context() - state = carrier.get("tracestate") - if state is not None and len(state.encode("utf-8")) > 512: - state = None - emit_event("telemetry.carrier.dropped", {"reason": "tracestate_capacity"}) - return { - "submission_traceparent": carrier.get("traceparent"), - "submission_tracestate": state, - } - - -def extract_context(carrier: Mapping[str, str]) -> Context: - """Use the SDK parser within the application's bounded, two-header carrier.""" - if not is_enabled(): - return Context() - bounded = { - key: value - for key in ("traceparent", "tracestate") - if (value := carrier.get(key)) is not None and len(value.encode("utf-8")) <= 512 - } - return PROPAGATOR.extract(bounded, context=Context()) - - -def submission_links(parent: str | None, state: str | None) -> tuple[Link, ...]: - carrier = {} - if parent is not None: - carrier["traceparent"] = parent - if state is not None: - carrier["tracestate"] = state - context = trace.get_current_span(extract_context(carrier)).get_span_context() - return (Link(context),) if context.is_valid else () - - class TelemetryMiddleware: """Observe complete HTTP responses without recording paths, headers or bodies.""" diff --git a/app/persistence/__init__.py b/app/persistence/__init__.py index e69de29b..3688a954 100644 --- a/app/persistence/__init__.py +++ b/app/persistence/__init__.py @@ -0,0 +1,3 @@ +"""Legacy import; use inkcre_core for new consumers.""" + +from inkcre_core.persistence import * diff --git a/app/persistence/agent/__init__.py b/app/persistence/agent/__init__.py index e69de29b..f3363a88 100644 --- a/app/persistence/agent/__init__.py +++ b/app/persistence/agent/__init__.py @@ -0,0 +1,3 @@ +"""Legacy import; use inkcre_core for new consumers.""" + +from inkcre_core.persistence.agent import * diff --git a/app/persistence/agent/repository.py b/app/persistence/agent/repository.py index 3c7ca031..1cd24d2f 100644 --- a/app/persistence/agent/repository.py +++ b/app/persistence/agent/repository.py @@ -1,49 +1,10 @@ -"""Persisted Agent definitions; Thread runtime state has a separate owner.""" +"""Legacy import; use inkcre_core for new consumers.""" -import sqlmodel -from sqlalchemy.ext.asyncio import AsyncSession -from app.schemas.agent import AgentDefinitionModel, AgentForm +from typing import TYPE_CHECKING +from importlib import import_module +import sys +sys.modules[__name__] = import_module("inkcre_core.persistence.agent.repository") -class AgentRepository: - def __init__(self, session: AsyncSession) -> None: - self._session = session - - async def get( - self, agent_id: int, *, for_update: bool = False - ) -> AgentDefinitionModel | None: - statement = sqlmodel.select(AgentDefinitionModel).where( - AgentDefinitionModel.id == agent_id - ) - if for_update: - statement = statement.with_for_update() - return (await self._session.scalars(statement)).one_or_none() - - async def list( - self, *, limit: int | None, cursor: int | None - ) -> tuple[list[AgentDefinitionModel], int | None]: - statement = sqlmodel.select(AgentDefinitionModel).order_by( - sqlmodel.col(AgentDefinitionModel.id) - ) - if cursor is not None: - statement = statement.where(sqlmodel.col(AgentDefinitionModel.id) > cursor) - if limit is not None: - statement = statement.limit(limit + 1) - rows = list((await self._session.scalars(statement)).all()) - more = limit is not None and len(rows) > limit - rows = rows[:limit] - return rows, rows[-1].id if more else None - - async def create(self, form: AgentForm) -> AgentDefinitionModel: - record = AgentDefinitionModel(**form.model_dump()) - self._session.add(record) - await self._session.flush() - return record - - async def save(self, record: AgentDefinitionModel) -> None: - self._session.add(record) - await self._session.flush() - await self._session.refresh(record) - - async def delete(self, record: AgentDefinitionModel) -> None: - await self._session.delete(record) +if TYPE_CHECKING: + from inkcre_core.persistence.agent.repository import * diff --git a/app/persistence/agent/uow.py b/app/persistence/agent/uow.py index f438b94a..ed952b06 100644 --- a/app/persistence/agent/uow.py +++ b/app/persistence/agent/uow.py @@ -1,13 +1,10 @@ -"""Short transaction for persisted Agent definitions.""" +"""Legacy import; use inkcre_core for new consumers.""" -from collections.abc import AsyncGenerator -from contextlib import asynccontextmanager +from typing import TYPE_CHECKING +from importlib import import_module +import sys -from app.engine import AsyncSessionFactory -from .repository import AgentRepository +sys.modules[__name__] = import_module("inkcre_core.persistence.agent.uow") - -@asynccontextmanager -async def agent_uow() -> AsyncGenerator[AgentRepository, None]: - async with AsyncSessionFactory.begin() as session: - yield AgentRepository(session) +if TYPE_CHECKING: + from inkcre_core.persistence.agent.uow import * diff --git a/app/persistence/ai/__init__.py b/app/persistence/ai/__init__.py index e69de29b..bab590fc 100644 --- a/app/persistence/ai/__init__.py +++ b/app/persistence/ai/__init__.py @@ -0,0 +1,3 @@ +"""Legacy import; use inkcre_core for new consumers.""" + +from inkcre_core.persistence.ai import * diff --git a/app/persistence/ai/repository.py b/app/persistence/ai/repository.py index b26272a4..4547bbb4 100644 --- a/app/persistence/ai/repository.py +++ b/app/persistence/ai/repository.py @@ -1,47 +1,10 @@ -"""AI registry facts inside the caller's transaction.""" +"""Legacy import; use inkcre_core for new consumers.""" -from collections.abc import Sequence -from typing import Any +from typing import TYPE_CHECKING +from importlib import import_module +import sys -import sqlmodel -from sqlalchemy.dialects.postgresql import insert -from sqlalchemy.ext.asyncio import AsyncSession +sys.modules[__name__] = import_module("inkcre_core.persistence.ai.repository") -from app.schemas.ai import AIDialectModel, AIModelModel, AIProviderModel - - -class AIRepository: - def __init__(self, session: AsyncSession) -> None: - self._session = session - - async def get_model(self, model_id: int) -> AIModelModel | None: - return await self._session.get(AIModelModel, model_id) - - async def get_provider(self, provider_id: int) -> AIProviderModel | None: - return await self._session.get(AIProviderModel, provider_id) - - async def list_models( - self, *, limit: int | None, cursor: int | None - ) -> tuple[list[AIModelModel], int | None]: - statement = sqlmodel.select(AIModelModel).order_by(sqlmodel.col(AIModelModel.id)) - if cursor is not None: - statement = statement.where(sqlmodel.col(AIModelModel.id) > cursor) - if limit is not None: - statement = statement.limit(limit + 1) - rows = list((await self._session.scalars(statement)).all()) - more = limit is not None and len(rows) > limit - rows = rows[:limit] - return rows, rows[-1].id if more else None - - async def sync_dialects(self, records: Sequence[dict[str, Any]]) -> None: - if not records: - return - statement = insert(AIDialectModel).values(records) - statement = statement.on_conflict_do_update( - index_elements=[AIDialectModel.id], - set_={ - "description": statement.excluded.description, - "config_schema": statement.excluded.config_schema, - }, - ) - await self._session.execute(statement) +if TYPE_CHECKING: + from inkcre_core.persistence.ai.repository import * diff --git a/app/persistence/ai/uow.py b/app/persistence/ai/uow.py index 0f1f40d5..d3abef8d 100644 --- a/app/persistence/ai/uow.py +++ b/app/persistence/ai/uow.py @@ -1,13 +1,10 @@ -"""Short database scope for AI facts; provider calls outlive this scope.""" +"""Legacy import; use inkcre_core for new consumers.""" -from collections.abc import AsyncGenerator -from contextlib import asynccontextmanager +from typing import TYPE_CHECKING +from importlib import import_module +import sys -from app.engine import AsyncSessionFactory -from .repository import AIRepository +sys.modules[__name__] = import_module("inkcre_core.persistence.ai.uow") - -@asynccontextmanager -async def ai_uow() -> AsyncGenerator[AIRepository, None]: - async with AsyncSessionFactory.begin() as session: - yield AIRepository(session) +if TYPE_CHECKING: + from inkcre_core.persistence.ai.uow import * diff --git a/app/persistence/cron/__init__.py b/app/persistence/cron/__init__.py index e69de29b..50253c5c 100644 --- a/app/persistence/cron/__init__.py +++ b/app/persistence/cron/__init__.py @@ -0,0 +1,3 @@ +"""Legacy import; use inkcre_core for new consumers.""" + +from inkcre_core.persistence.cron import * diff --git a/app/persistence/cron/repository.py b/app/persistence/cron/repository.py index 06b43f00..4e8ce3bd 100644 --- a/app/persistence/cron/repository.py +++ b/app/persistence/cron/repository.py @@ -1,50 +1,10 @@ -"""Locked Cron occurrences and database-clock observations.""" +"""Legacy import; use inkcre_core for new consumers.""" -import datetime +from typing import TYPE_CHECKING +from importlib import import_module +import sys -import sqlmodel -from sqlalchemy.ext.asyncio import AsyncSession +sys.modules[__name__] = import_module("inkcre_core.persistence.cron.repository") -from app.schemas.cron import CronModel - - -class CronRepository: - def __init__(self, session: AsyncSession) -> None: - self._session = session - - async def get(self, cron_id: int, *, lock: bool = False, skip_locked: bool = False): - statement = sqlmodel.select(CronModel).where(CronModel.id == cron_id) - if lock: - statement = statement.with_for_update(skip_locked=skip_locked) - return (await self._session.scalars(statement)).one_or_none() - - async def database_now(self) -> datetime.datetime: - return ( - await self._session.execute(sqlmodel.select(sqlmodel.func.statement_timestamp())) - ).scalar_one() - - async def enabled_ids(self): - return tuple( - await self._session.scalars( - sqlmodel.select(CronModel.id).where(sqlmodel.col(CronModel.enabled).is_(True)) - ) - ) - - async def list(self, *, limit: int | None, cursor: int | None): - statement = sqlmodel.select(CronModel).order_by(sqlmodel.col(CronModel.id)) - if cursor is not None: - statement = statement.where(sqlmodel.col(CronModel.id) > cursor) - if limit is not None: - statement = statement.limit(limit + 1) - rows = list((await self._session.scalars(statement)).all()) - more = limit is not None and len(rows) > limit - rows = rows[:limit] - return rows, rows[-1].id if more else None - - async def save(self, cron: CronModel) -> None: - self._session.add(cron) - await self._session.flush() - await self._session.refresh(cron) - - async def delete(self, cron: CronModel) -> None: - await self._session.delete(cron) +if TYPE_CHECKING: + from inkcre_core.persistence.cron.repository import * diff --git a/app/persistence/cron/uow.py b/app/persistence/cron/uow.py index 23a96dd2..6d2cf1cd 100644 --- a/app/persistence/cron/uow.py +++ b/app/persistence/cron/uow.py @@ -1,24 +1,10 @@ -"""A Cron occurrence and its admitted Job commit together.""" +"""Legacy import; use inkcre_core for new consumers.""" -from collections.abc import AsyncGenerator -from contextlib import asynccontextmanager -from dataclasses import dataclass +from typing import TYPE_CHECKING +from importlib import import_module +import sys -from app.engine import AsyncSessionFactory -from app.persistence.job.repository import JobRepository -from app.persistence.job.uow import JobUnitOfWork -from app.persistence.source.repository import SourceRepository -from .repository import CronRepository +sys.modules[__name__] = import_module("inkcre_core.persistence.cron.uow") - -@dataclass(frozen=True) -class CronUnitOfWork(JobUnitOfWork): - crons: CronRepository - - -@asynccontextmanager -async def cron_uow() -> AsyncGenerator[CronUnitOfWork, None]: - async with AsyncSessionFactory.begin() as session: - yield CronUnitOfWork( - JobRepository(session), SourceRepository(session), CronRepository(session) - ) +if TYPE_CHECKING: + from inkcre_core.persistence.cron.uow import * diff --git a/app/persistence/deployment_config/__init__.py b/app/persistence/deployment_config/__init__.py index e69de29b..ceeb82fa 100644 --- a/app/persistence/deployment_config/__init__.py +++ b/app/persistence/deployment_config/__init__.py @@ -0,0 +1,3 @@ +"""Legacy import; use inkcre_core for new consumers.""" + +from inkcre_core.persistence.deployment_config import * diff --git a/app/persistence/deployment_config/repository.py b/app/persistence/deployment_config/repository.py index fe1d7498..90a8112c 100644 --- a/app/persistence/deployment_config/repository.py +++ b/app/persistence/deployment_config/repository.py @@ -1,68 +1,12 @@ -"""Deployment config persistence and transaction composition.""" +"""Legacy import; use inkcre_core for new consumers.""" -from typing import Any +from typing import TYPE_CHECKING +from importlib import import_module +import sys -import sqlmodel -from sqlalchemy.dialects.postgresql import insert -from sqlalchemy.ext.asyncio import AsyncSession +sys.modules[__name__] = import_module( + "inkcre_core.persistence.deployment_config.repository" +) -from app.schemas.deployment_config import DeploymentConfigModel - - -class DeploymentConfigRepository: - def __init__(self, session: AsyncSession) -> None: - self._session = session - - async def get( - self, key: str, *, for_update: bool = False - ) -> DeploymentConfigModel | None: - statement = sqlmodel.select(DeploymentConfigModel).where( - DeploymentConfigModel.key == key - ) - if for_update: - statement = statement.with_for_update() - return (await self._session.scalars(statement)).one_or_none() - - async def list( - self, *, limit: int | None, cursor: str | None - ) -> tuple[list[DeploymentConfigModel], str | None]: - statement = sqlmodel.select(DeploymentConfigModel).order_by(DeploymentConfigModel.key) - if cursor is not None: - statement = statement.where(DeploymentConfigModel.key > cursor) - if limit is not None: - statement = statement.limit(limit + 1) - rows = list((await self._session.scalars(statement)).all()) - more = limit is not None and len(rows) > limit - rows = rows[:limit] - return rows, rows[-1].key if more else None - - async def replace( - self, key: str, schema_id: str, value: dict[str, Any] - ) -> tuple[DeploymentConfigModel, bool]: - statement = insert(DeploymentConfigModel).values( - key=key, schema_id=schema_id, value=value - ) - statement = statement.on_conflict_do_nothing( - index_elements=[DeploymentConfigModel.key] - ).returning(DeploymentConfigModel) - record = (await self._session.scalars(statement)).one_or_none() - if record is not None: - return record, True - # The insert conflict, rather than a preceding read, determines creation status. - statement = ( - sqlmodel.update(DeploymentConfigModel) - .where(sqlmodel.col(DeploymentConfigModel.key) == key) - .values(schema_id=schema_id, value=value) - .returning(DeploymentConfigModel) - ) - record = (await self._session.scalars(statement)).one() - return record, False - - async def save(self, record: DeploymentConfigModel) -> None: - self._session.add(record) - await self._session.flush() - # UPDATE timestamps are managed by database triggers, not Python defaults. - await self._session.refresh(record) - - async def delete(self, record: DeploymentConfigModel) -> None: - await self._session.delete(record) +if TYPE_CHECKING: + from inkcre_core.persistence.deployment_config.repository import * diff --git a/app/persistence/deployment_config/uow.py b/app/persistence/deployment_config/uow.py index 425a9c15..aa0968a6 100644 --- a/app/persistence/deployment_config/uow.py +++ b/app/persistence/deployment_config/uow.py @@ -1,12 +1,10 @@ -"""Deployment configuration transaction composition.""" +"""Legacy import; use inkcre_core for new consumers.""" -from collections.abc import AsyncGenerator -from contextlib import asynccontextmanager -from app.engine import AsyncSessionFactory -from .repository import DeploymentConfigRepository +from typing import TYPE_CHECKING +from importlib import import_module +import sys +sys.modules[__name__] = import_module("inkcre_core.persistence.deployment_config.uow") -@asynccontextmanager -async def configuration_transaction() -> AsyncGenerator[DeploymentConfigRepository, None]: - async with AsyncSessionFactory.begin() as session: - yield DeploymentConfigRepository(session) +if TYPE_CHECKING: + from inkcre_core.persistence.deployment_config.uow import * diff --git a/app/persistence/extension/__init__.py b/app/persistence/extension/__init__.py index e69de29b..251f0360 100644 --- a/app/persistence/extension/__init__.py +++ b/app/persistence/extension/__init__.py @@ -0,0 +1,3 @@ +"""Legacy import; use inkcre_core for new consumers.""" + +from inkcre_core.persistence.extension import * diff --git a/app/persistence/extension/repository.py b/app/persistence/extension/repository.py index 92107b1d..c7b54c3b 100644 --- a/app/persistence/extension/repository.py +++ b/app/persistence/extension/repository.py @@ -1,72 +1,10 @@ -"""Session-bound access to canonical Extension rows and the enabled RPC.""" +"""Legacy import; use inkcre_core for new consumers.""" -import uuid +from typing import TYPE_CHECKING +from importlib import import_module +import sys -import sqlalchemy -import sqlmodel -from sqlalchemy.ext.asyncio import AsyncSession +sys.modules[__name__] = import_module("inkcre_core.persistence.extension.repository") -from app.schemas.extension import ExtensionModel, ExtensionPeerParticipationModel - - -class ExtensionRepository: - def __init__(self, session: AsyncSession) -> None: - self._session = session - - async def list(self): - rows = await self._session.scalars( - sqlmodel.select(ExtensionModel).order_by(ExtensionModel.name) - ) - return tuple(rows) - - async def get(self, name: str, *, lock: bool = False): - statement = sqlmodel.select(ExtensionModel).where(ExtensionModel.name == name) - if lock: - statement = statement.with_for_update() - return (await self._session.scalars(statement)).one_or_none() - - async def try_install_lock(self, name: str) -> bool: - result = await self._session.execute( - sqlalchemy.text("SELECT pg_try_advisory_xact_lock(hashtextextended(:name, 0))"), - {"name": name}, - ) - return bool(result.scalar_one()) - - async def save(self, row: ExtensionModel) -> None: - self._session.add(row) - await self._session.flush() - - async def delete(self, row: ExtensionModel) -> None: - await self._session.delete(row) - - async def list_participations(self, peer_id: uuid.UUID): - rows = await self._session.scalars( - sqlmodel.select(ExtensionPeerParticipationModel) - .where(ExtensionPeerParticipationModel.peer_id == peer_id) - .order_by(ExtensionPeerParticipationModel.name) - ) - return tuple(rows) - - async def list_participations_for_extension(self, name: str): - rows = await self._session.scalars( - sqlmodel.select(ExtensionPeerParticipationModel).where( - ExtensionPeerParticipationModel.name == name - ) - ) - return tuple(rows) - - async def get_participation(self, name: str, peer_id: uuid.UUID, *, lock: bool = False): - statement = sqlmodel.select(ExtensionPeerParticipationModel).where( - ExtensionPeerParticipationModel.name == name, - ExtensionPeerParticipationModel.peer_id == peer_id, - ) - if lock: - statement = statement.with_for_update() - return (await self._session.scalars(statement)).one_or_none() - - async def save_participation(self, row: ExtensionPeerParticipationModel) -> None: - self._session.add(row) - await self._session.flush() - - async def delete_participation(self, row: ExtensionPeerParticipationModel) -> None: - await self._session.delete(row) +if TYPE_CHECKING: + from inkcre_core.persistence.extension.repository import * diff --git a/app/persistence/extension/uow.py b/app/persistence/extension/uow.py index 2cefad01..903afc73 100644 --- a/app/persistence/extension/uow.py +++ b/app/persistence/extension/uow.py @@ -1,13 +1,10 @@ -"""Short Extension transaction; publication and acquisition remain outside.""" +"""Legacy import; use inkcre_core for new consumers.""" -from collections.abc import AsyncGenerator -from contextlib import asynccontextmanager +from typing import TYPE_CHECKING +from importlib import import_module +import sys -from app.engine import AsyncSessionFactory -from .repository import ExtensionRepository +sys.modules[__name__] = import_module("inkcre_core.persistence.extension.uow") - -@asynccontextmanager -async def extension_uow() -> AsyncGenerator[ExtensionRepository, None]: - async with AsyncSessionFactory.begin() as session: - yield ExtensionRepository(session) +if TYPE_CHECKING: + from inkcre_core.persistence.extension.uow import * diff --git a/app/persistence/info_base/__init__.py b/app/persistence/info_base/__init__.py index e69de29b..69343b5f 100644 --- a/app/persistence/info_base/__init__.py +++ b/app/persistence/info_base/__init__.py @@ -0,0 +1,3 @@ +"""Legacy import; use inkcre_core for new consumers.""" + +from inkcre_core.persistence.info_base import * diff --git a/app/persistence/info_base/repository.py b/app/persistence/info_base/repository.py index 282bed81..b8b0cbe1 100644 --- a/app/persistence/info_base/repository.py +++ b/app/persistence/info_base/repository.py @@ -1,501 +1,10 @@ -"""Graph persistence within a caller-owned asynchronous transaction.""" +"""Legacy import; use inkcre_core for new consumers.""" -from collections.abc import Iterable -import random -import typing -from typing import Optional as Opt -import sqlalchemy -import sqlmodel -from utils.types_ import Undefined, _undefined -from app.schemas.info_base.block import BlockID, ResolverType -from app.schemas.info_base.relation import RelationID -from app.schemas.info_base.storage import StorageID +from typing import TYPE_CHECKING +from importlib import import_module +import sys -from sqlalchemy.ext.asyncio import AsyncSession +sys.modules[__name__] = import_module("inkcre_core.persistence.info_base.repository") -from app.schemas.info_base.block import BlockForm, BlockModel -from app.schemas.info_base.relation import RelationCreateForm, RelationModel - - -class BlockRepository: - def __init__(self, session: AsyncSession) -> None: - self._session = session - - async def create_many(self, forms: Iterable[BlockForm]) -> tuple[BlockModel, ...]: - """Insert producer fields and load database identities in one flush.""" - blocks = tuple( - BlockModel.model_validate(form.model_dump(include=set(BlockForm.model_fields))) - for form in forms - ) - if blocks: - self._session.add_all(blocks) - await self._session.flush() - return blocks - - async def get_many(self, block_ids: typing.Collection[BlockID]) -> tuple[BlockModel, ...]: - """Return the existing Blocks from a bounded identity set.""" - if not block_ids: - return () - return tuple( - ( - await self._session.scalars( - sqlmodel.select(BlockModel).where( - sqlmodel.col(BlockModel.id).in_(tuple(block_ids)) - ) - ) - ).all() - ) - - async def get_by_resolvers( - self, resolvers: typing.Collection[str], *, lock: bool = False - ) -> tuple[BlockModel, ...]: - if not resolvers: - return () - statement = sqlmodel.select(BlockModel).where( - sqlmodel.col(BlockModel.resolver).in_(tuple(resolvers)) - ) - if lock: - statement = statement.order_by(sqlmodel.col(BlockModel.id)).with_for_update() - return tuple(await self._session.scalars(statement)) - - async def save_many(self, blocks: Iterable[BlockModel]) -> None: - self._session.add_all(blocks) - await self._session.flush() - - async def get_random_many(self, count: int) -> tuple[BlockModel, ...]: - """Return up to count distinct random Blocks.""" - if count <= 0: - return () - return tuple( - ( - await self._session.scalars( - sqlmodel.select(BlockModel).order_by(sqlmodel.func.random()).limit(count) - ) - ).all() - ) - - async def get_random(self) -> Opt[BlockModel]: - """Return one existing Block without transferring all identities.""" - block_id_column = sqlmodel.col(BlockModel.id) - count = ( - await self._session.scalars(sqlmodel.select(sqlmodel.func.count(block_id_column))) - ).one() - if count == 0: - return None - offset = random.SystemRandom().randrange(count) - return ( - await self._session.scalars( - sqlmodel.select(BlockModel).order_by(block_id_column).offset(offset).limit(1) - ) - ).one() - - async def get(self, block_id: BlockID, *, lock: bool = False) -> Opt[BlockModel]: - statement = sqlmodel.select(BlockModel).where(BlockModel.id == block_id) - if lock: - statement = statement.with_for_update() - return (await self._session.scalars(statement)).one_or_none() - - async def save(self, block: BlockModel) -> None: - self._session.add(block) - await self._session.flush() - - async def edit_block( - self, - block_id: BlockID, - content: Opt[str] = None, - resolver: Opt[ResolverType] = None, - storage: Opt[StorageID] | Undefined = _undefined, - ) -> BlockModel: - """编辑块""" - block = await self.get(block_id) - if block is None: - raise ValueError("Block not found") - if content is not None: - block.content = content - if resolver is not None: - block.resolver = resolver - if storage is not _undefined: - block.storage = typing.cast(StorageID | None, storage) - self._session.add(block) - await self._session.flush() - await self._session.refresh(block) - return block - - async def delete(self, block_id: BlockID) -> bool: - """Delete a block, within the surrounding transaction.""" - block = await self.get(block_id) - if block is None: - return False - await self._session.delete(block) - await self._session.flush() - return True - - async def get_recent( - self, num: int = 10, resolver: Opt[ResolverType] = None - ) -> tuple[BlockModel, ...]: - statement = ( - sqlmodel.select(BlockModel).order_by(sqlmodel.desc(BlockModel.created_at)).limit(num) - ) - if resolver: - statement = statement.where(BlockModel.resolver == resolver) - return tuple((await self._session.scalars(statement)).all()) - - async def create(self, form: BlockForm) -> BlockModel: - return (await self.create_many((form,)))[0] - - async def find_content(self, resolver: str, content: str) -> BlockModel | None: - return ( - await self._session.scalars( - sqlmodel.select(BlockModel).where( - BlockModel.resolver == resolver, - BlockModel.content == content, - ) - ) - ).one_or_none() - - async def find_json_field( - self, - resolver: str, - field: str, - value: str, - *, - block_ids: typing.Collection[BlockID] | None = None, - ) -> tuple[BlockModel, ...]: - from utils.sql import find_by_json_field - - statement = sqlmodel.select(BlockModel).where( - BlockModel.resolver == resolver, - find_by_json_field(BlockModel.content, field, value), - ) - if block_ids is not None: - statement = statement.where(sqlmodel.col(BlockModel.id).in_(tuple(block_ids))) - return tuple(await self._session.scalars(statement)) - - async def find_json_field_many( - self, - resolvers: typing.Collection[str], - field: str, - values: typing.Collection[str], - ) -> tuple[BlockModel, ...]: - """Find inline JSON records in a bounded native-identity set.""" - from sqlalchemy.dialects.postgresql import JSONB - - if not resolvers or not values: - return () - return tuple( - await self._session.scalars( - sqlmodel.select(BlockModel) - .where( - sqlmodel.col(BlockModel.resolver).in_(tuple(resolvers)), - sqlalchemy.case( - ( - sqlalchemy.func.pg_input_is_valid(BlockModel.content, "jsonb"), - sqlalchemy.cast(BlockModel.content, JSONB)[field].astext, - ) - ).in_(tuple(values)), - sqlmodel.col(BlockModel.storage).is_(None), - ) - .order_by(sqlmodel.col(BlockModel.id)) - ) - ) - - async def acquire_transaction_lock(self, namespace: str, *, shared: bool = False) -> None: - """Serialize an owning use case until its surrounding transaction ends.""" - function = ( - sqlmodel.func.pg_advisory_xact_lock_shared - if shared - else sqlmodel.func.pg_advisory_xact_lock - ) - await self._session.execute( - sqlmodel.select(function(sqlmodel.func.hashtextextended(namespace, 0))) - ) - - async def get_related( - self, block_id: BlockID, *, content: str, outgoing: bool = True - ) -> BlockModel | None: - endpoint = RelationModel.from_ if outgoing else RelationModel.to_ - other = RelationModel.to_ if outgoing else RelationModel.from_ - statement = ( - sqlmodel.select(BlockModel) - .join(RelationModel, sqlmodel.col(BlockModel.id) == other) - .where( - endpoint == block_id, - RelationModel.content == content, - ) - .limit(1) - ) - return (await self._session.scalars(statement)).first() - - async def select_ids( - self, - limit: int, - *, - exclude_resolvers: typing.Collection[str], - random_order: bool = False, - ) -> tuple[BlockID, ...]: - statement = sqlmodel.select(BlockModel.id).where( - sqlmodel.col(BlockModel.resolver).not_in(tuple(exclude_resolvers)) - ) - statement = ( - statement.order_by(sqlalchemy.func.random()) - if random_order - else statement.order_by( - sqlmodel.desc(BlockModel.updated_at), sqlmodel.desc(BlockModel.id) - ) - ) - return tuple( - typing.cast(BlockID, value) - for value in await self._session.scalars(statement.limit(limit)) - ) - - async def matching_content(self, resolver: str, content: str) -> tuple[BlockModel, ...]: - return tuple( - await self._session.scalars( - sqlmodel.select(BlockModel).where( - BlockModel.resolver == resolver, BlockModel.content == content - ) - ) - ) - - async def without_outgoing_relation( - self, resolvers: typing.Collection[str], content: str, *, limit: int - ) -> tuple[BlockModel, ...]: - exists = sqlalchemy.exists( - sqlmodel.select(RelationModel.id).where( - RelationModel.from_ == BlockModel.id, RelationModel.content == content - ) - ) - return tuple( - await self._session.scalars( - sqlmodel.select(BlockModel) - .where(sqlmodel.col(BlockModel.resolver).in_(tuple(resolvers)), ~exists) - .order_by(sqlmodel.col(BlockModel.id)) - .limit(limit) - ) - ) - - -class RelationRepository: - def __init__(self, session: AsyncSession) -> None: - self._session = session - - async def create_many( - self, forms: Iterable[RelationCreateForm] - ) -> tuple[RelationModel, ...]: - """Insert directed relations without committing the surrounding operation.""" - relations = tuple(RelationModel.model_validate(form) for form in forms) - if relations: - self._session.add_all(relations) - await self._session.flush() - return relations - - async def get_many( - self, relation_ids: typing.Collection[RelationID] - ) -> tuple[RelationModel, ...]: - """Return the existing Relations from a bounded identity set.""" - if not relation_ids: - return () - return tuple( - ( - await self._session.scalars( - sqlmodel.select(RelationModel).where( - sqlmodel.col(RelationModel.id).in_(tuple(relation_ids)) - ) - ) - ).all() - ) - - async def get_for_endpoints( - self, block_ids: typing.Collection[BlockID], *, contents: typing.Collection[str] - ) -> tuple[RelationModel, ...]: - if not block_ids: - return () - return tuple( - await self._session.scalars( - sqlmodel.select(RelationModel).where( - sqlmodel.col(RelationModel.content).in_(tuple(contents)), - sqlalchemy.or_( - sqlmodel.col(RelationModel.from_).in_(tuple(block_ids)), - sqlmodel.col(RelationModel.to_).in_(tuple(block_ids)), - ), - ) - ) - ) - - async def delete_many(self, relation_ids: typing.Collection[RelationID]) -> None: - if relation_ids: - await self._session.execute( - sqlalchemy.delete(RelationModel).where( - sqlmodel.col(RelationModel.id).in_(tuple(relation_ids)) - ) - ) - - async def get_outgoing_many( - self, - block_ids: typing.Collection[BlockID], - *, - to_ids: typing.Collection[BlockID] | None = None, - content: str | None = None, - lock: bool = False, - ) -> tuple[RelationModel, ...]: - if not block_ids: - return () - statement = sqlmodel.select(RelationModel).where( - sqlmodel.col(RelationModel.from_).in_(tuple(block_ids)) - ) - if to_ids is not None: - statement = statement.where(sqlmodel.col(RelationModel.to_).in_(tuple(to_ids))) - if content is not None: - statement = statement.where(RelationModel.content == content) - if lock: - statement = statement.order_by(sqlmodel.col(RelationModel.id)).with_for_update() - return tuple(await self._session.scalars(statement)) - - async def save(self, relation: RelationModel) -> None: - self._session.add(relation) - await self._session.flush() - - async def remove(self, relation: RelationModel) -> None: - await self._session.delete(relation) - - async def get_by_id(self, relation_id: RelationID) -> RelationModel | None: - return await self._session.get(RelationModel, relation_id) - - async def get_endpoint_page( - self, - block_ids: typing.Collection[BlockID], - *, - endpoint: typing.Literal["from", "to"], - contents: typing.Collection[str] = (), - cursor: RelationID | None = None, - limit: int = 21, - ) -> tuple[RelationModel, ...]: - """Read a bounded Relation page for one persisted endpoint direction.""" - if not block_ids or limit <= 0: - return () - endpoint_column = RelationModel.from_ if endpoint == "from" else RelationModel.to_ - statement = sqlmodel.select(RelationModel).where( - sqlmodel.col(endpoint_column).in_(tuple(block_ids)) - ) - if contents: - statement = statement.where(sqlmodel.col(RelationModel.content).in_(tuple(contents))) - relation_id_column = sqlmodel.col(RelationModel.id) - if cursor is not None: - statement = statement.where(relation_id_column < cursor) - statement = statement.order_by(sqlmodel.desc(relation_id_column)).limit(limit) - return tuple((await self._session.scalars(statement)).all()) - - async def fetchsert(self, relation: RelationModel) -> RelationModel: - """Create if not exists, else return the existing one. - - Will NOT commit the session. - """ - existing = ( - await self._session.scalars( - sqlmodel.select(RelationModel).where( - RelationModel.content == relation.content, - RelationModel.from_ == relation.from_, - RelationModel.to_ == relation.to_, - ) - ) - ).one_or_none() - if existing is not None: - return existing - self._session.add(relation) - await self._session.flush() - return relation - - async def get( - self, - block_id: BlockID, - include_in: bool = True, - include_out: bool = True, - content: Opt[str] = None, - ) -> tuple[RelationModel, ...]: - """Get relations from/to a block - - :param include_in: Include the relations where the block is the target - :param include_out: Include the relations where the block is the source - :param content: If specified, filter relations by content (eq) - """ - if not include_in and (not include_out): - return () - directions: list[typing.Any] = [] - if include_in: - directions.append(RelationModel.to_ == block_id) - if include_out: - directions.append(RelationModel.from_ == block_id) - statement = sqlmodel.select(RelationModel).where(sqlalchemy.or_(*directions)) - if content is not None: - statement = statement.where(RelationModel.content == content) - res = (await self._session.scalars(statement)).all() - return tuple(res) - - async def update( - self, - relation_id: RelationID, - *, - from_: BlockID | Undefined = _undefined, - to_: BlockID | Undefined = _undefined, - content: str | Undefined = _undefined, - ) -> RelationModel: - """Update selected relation facts in the surrounding transaction.""" - relation = ( - await self._session.scalars( - sqlmodel.select(RelationModel).where(RelationModel.id == relation_id) - ) - ).one_or_none() - if relation is None: - raise ValueError("Relation not found") - if from_ is not _undefined: - relation.from_ = typing.cast(BlockID, from_) - if to_ is not _undefined: - relation.to_ = typing.cast(BlockID, to_) - if content is not _undefined: - relation.content = typing.cast(str, content) - self._session.add(relation) - await self._session.flush() - await self._session.refresh(relation) - return relation - - async def delete(self, relation_id: RelationID) -> bool: - """Delete a relation, within the surrounding transaction.""" - relation = ( - await self._session.scalars( - sqlmodel.select(RelationModel).where(RelationModel.id == relation_id) - ) - ).one_or_none() - if relation is None: - return False - await self._session.delete(relation) - await self._session.flush() - return True - - async def create(self, from_: BlockID, to_: BlockID, content: str) -> RelationModel: - return ( - await self.create_many((RelationCreateForm(from_=from_, to_=to_, content=content),)) - )[0] - - async def random_incoming_sources( - self, block_id: BlockID, content: str, limit: int - ) -> tuple[BlockID, ...]: - return tuple( - await self._session.scalars( - sqlmodel.select(RelationModel.from_) - .where(RelationModel.to_ == block_id, RelationModel.content == content) - .order_by(sqlalchemy.func.random()) - .limit(limit) - ) - ) - - async def recent( - self, limit: int, *, contents: typing.Collection[str] = () - ) -> tuple[RelationModel, ...]: - statement = ( - sqlmodel.select(RelationModel) - .order_by(sqlmodel.desc(RelationModel.updated_at), sqlmodel.desc(RelationModel.id)) - .limit(limit) - ) - if contents: - statement = statement.where(sqlmodel.col(RelationModel.content).in_(tuple(contents))) - return tuple(await self._session.scalars(statement)) +if TYPE_CHECKING: + from inkcre_core.persistence.info_base.repository import * diff --git a/app/persistence/info_base/storage.py b/app/persistence/info_base/storage.py index 445be112..cd852c37 100644 --- a/app/persistence/info_base/storage.py +++ b/app/persistence/info_base/storage.py @@ -1,92 +1,10 @@ -"""Storage catalog and byte persistence bound to the enclosing graph transaction.""" +"""Legacy import; use inkcre_core for new consumers.""" -from collections.abc import Sequence -from typing import Any -from uuid import UUID +from typing import TYPE_CHECKING +from importlib import import_module +import sys -import sqlalchemy -from sqlalchemy.dialects.postgresql import insert -from sqlalchemy.ext.asyncio import AsyncSession -import sqlmodel +sys.modules[__name__] = import_module("inkcre_core.persistence.info_base.storage") -from app.schemas.info_base.storage import StorageBlobModel, StorageModel, StorageTypesModel - - -class StorageRepository: - def __init__(self, session: AsyncSession) -> None: - self._session = session - - async def get(self, storage_id: int) -> StorageModel: - return ( - await self._session.scalars( - sqlmodel.select(StorageModel).where(StorageModel.id == storage_id) - ) - ).one() - - async def sync_types(self, values: Sequence[dict[str, Any]]) -> None: - if not values: - return - statement = insert(StorageTypesModel).values(values) - await self._session.execute( - statement.on_conflict_do_update( - index_elements=[StorageTypesModel.id], - set_={ - "description": statement.excluded.description, - "config_schema": statement.excluded.config_schema, - "writable": statement.excluded.writable, - }, - ) - ) - - async def sync_builtins(self, values: Sequence[dict[str, Any]]) -> None: - if not values: - return - statement = insert(StorageModel).values(values) - await self._session.execute( - statement.on_conflict_do_update( - index_elements=["id"], - set_={ - "type": statement.excluded.type, - "nickname": statement.excluded.nickname, - "config": statement.excluded.config, - }, - ) - ) - - async def create(self, storage: StorageModel) -> StorageModel: - self._session.add(storage) - await self._session.flush() - return storage - - async def read_blob(self, blob_id: UUID, *, lock: bool = False) -> bytes | None: - statement = sqlmodel.select(StorageBlobModel.data).where(StorageBlobModel.id == blob_id) - if lock: - statement = statement.with_for_update() - return (await self._session.scalars(statement)).one_or_none() - - async def create_blob(self, content: bytes) -> UUID: - blob = StorageBlobModel(data=content) - self._session.add(blob) - await self._session.flush() - return blob.id - - async def update_blob(self, blob_id: UUID, content: bytes) -> bool: - result = await self._session.scalars( - sqlalchemy.update(StorageBlobModel) - .where( - sqlmodel.col(StorageBlobModel.id) == blob_id, - ) - .values(data=content) - .returning(sqlmodel.col(StorageBlobModel.id)) - ) - return result.one_or_none() is not None - - async def delete_blob(self, blob_id: UUID) -> bool: - result = await self._session.scalars( - sqlalchemy.delete(StorageBlobModel) - .where( - sqlmodel.col(StorageBlobModel.id) == blob_id, - ) - .returning(sqlmodel.col(StorageBlobModel.id)) - ) - return result.one_or_none() is not None +if TYPE_CHECKING: + from inkcre_core.persistence.info_base.storage import * diff --git a/app/persistence/info_base/uow.py b/app/persistence/info_base/uow.py index 112e65ed..66285a0d 100644 --- a/app/persistence/info_base/uow.py +++ b/app/persistence/info_base/uow.py @@ -1,30 +1,10 @@ -"""Graph transaction composition; repositories share one private session.""" +"""Legacy import; use inkcre_core for new consumers.""" -from collections.abc import AsyncGenerator -from contextlib import asynccontextmanager -from dataclasses import dataclass +from typing import TYPE_CHECKING +from importlib import import_module +import sys -from app.engine import AsyncSessionFactory +sys.modules[__name__] = import_module("inkcre_core.persistence.info_base.uow") -from .repository import BlockRepository, RelationRepository -from .storage import StorageRepository - - -@dataclass(frozen=True) -class GraphUnitOfWork: - blocks: BlockRepository - relations: RelationRepository - storage: StorageRepository - - -@asynccontextmanager -async def graph_uow( - *, repeatable_read: bool = False -) -> AsyncGenerator[GraphUnitOfWork, None]: - """Commit on success; roll back on failure and always close the session.""" - async with AsyncSessionFactory.begin() as session: - if repeatable_read: - await session.connection(execution_options={"isolation_level": "REPEATABLE READ"}) - yield GraphUnitOfWork( - BlockRepository(session), RelationRepository(session), StorageRepository(session) - ) +if TYPE_CHECKING: + from inkcre_core.persistence.info_base.uow import * diff --git a/app/persistence/job/__init__.py b/app/persistence/job/__init__.py index e69de29b..0d8afbc4 100644 --- a/app/persistence/job/__init__.py +++ b/app/persistence/job/__init__.py @@ -0,0 +1,3 @@ +"""Legacy import; use inkcre_core for new consumers.""" + +from inkcre_core.persistence.job import * diff --git a/app/persistence/job/repository.py b/app/persistence/job/repository.py index 85862415..fbb8a655 100644 --- a/app/persistence/job/repository.py +++ b/app/persistence/job/repository.py @@ -1,140 +1,10 @@ -"""Session-bound durable Job catalog, claim and terminal writes.""" +"""Legacy import; use inkcre_core for new consumers.""" -import typing +from typing import TYPE_CHECKING +from importlib import import_module +import sys -import sqlalchemy -from sqlalchemy.dialects.postgresql import insert -from sqlalchemy.ext.asyncio import AsyncSession -import sqlmodel +sys.modules[__name__] = import_module("inkcre_core.persistence.job.repository") -from app.schemas.job import JobModel, JobStatus, JobTypeModel - - -class JobRepository: - def __init__(self, session: AsyncSession) -> None: - self._session = session - - async def sync_types(self, rows: list[dict]) -> None: - if not rows: - return - statement = insert(JobTypeModel).values(rows) - await self._session.execute( - statement.on_conflict_do_update( - index_elements=[JobTypeModel.id], - set_={ - key: getattr(statement.excluded, key) - for key in ("description", "parameters_schema", "default_timeout_seconds") - }, - ) - ) - - async def get(self, job_id: int) -> JobModel | None: - return await self._session.get(JobModel, job_id) - - async def get_type(self, type_: str) -> JobTypeModel | None: - return await self._session.get(JobTypeModel, type_) - - async def create(self, job: JobModel) -> JobModel: - self._session.add(job) - await self._session.flush() - await self._session.refresh(job) - return job - - async def list_types(self, *, limit: int | None, cursor: str | None): - statement = sqlmodel.select(JobTypeModel).order_by(JobTypeModel.id) - if cursor is not None: - statement = statement.where(JobTypeModel.id > cursor) - if limit is not None: - statement = statement.limit(limit + 1) - rows = list((await self._session.scalars(statement)).all()) - more = limit is not None and len(rows) > limit - rows = rows[:limit] - return rows, rows[-1].id if more else None - - async def list_jobs( - self, *, limit: int, cursor: int | None, type_: str | None, status: JobStatus | None - ): - statement = sqlmodel.select(JobModel).order_by(sqlmodel.col(JobModel.id).desc()) - if cursor is not None: - statement = statement.where(sqlmodel.col(JobModel.id) < cursor) - if type_ is not None: - statement = statement.where(JobModel.type == type_) - if status is not None: - statement = statement.where(JobModel.status == status) - rows = list((await self._session.scalars(statement.limit(limit + 1))).all()) - more = len(rows) > limit - rows = rows[:limit] - return rows, rows[-1].id if more else None - - async def abort(self, job_id: int) -> JobModel | None: - table = typing.cast(typing.Any, getattr(JobModel, "__table__")) - await self._session.execute( - sqlalchemy.update(table) - .where( - table.c.id == job_id, - table.c.status.in_((JobStatus.PENDING, JobStatus.RUNNING)), - ) - .values( - abort_requested=True, - status=sqlalchemy.case( - ( - table.c.status == JobStatus.PENDING, - sqlalchemy.cast(JobStatus.ABORTED.value, table.c.status.type), - ), - else_=table.c.status, - ), - ) - ) - return await self.get(job_id) - - async def abort_requests(self, active_ids: tuple[int, ...]): - return tuple( - await self._session.scalars( - sqlmodel.select(JobModel.id).where( - sqlmodel.col(JobModel.id).in_(active_ids), - sqlmodel.col(JobModel.abort_requested).is_(True), - ) - ) - ) - - async def claim(self, job_id: int) -> JobModel | None: - result = await self._session.scalars( - sqlalchemy.update(JobModel) - .where( - sqlmodel.col(JobModel.id) == job_id, - sqlmodel.col(JobModel.status) == JobStatus.PENDING, - ) - .values(status=JobStatus.RUNNING) - .returning(JobModel) - ) - return result.one_or_none() - - async def close(self, job: JobModel, status: JobStatus) -> bool: - table = typing.cast(typing.Any, getattr(JobModel, "__table__")) - result = await self._session.execute( - sqlalchemy.update(table) - .where(table.c.id == job.id, table.c.status == JobStatus.RUNNING) - .values(status=status, state=job.state) - ) - return bool(typing.cast(typing.Any, result).rowcount) - - async def expire_overdue(self) -> int: - table = typing.cast(typing.Any, getattr(JobModel, "__table__")) - result = await self._session.execute( - sqlalchemy.update(table) - .where( - table.c.status == JobStatus.RUNNING, - sqlalchemy.text( - "started_at + timeout_seconds * interval '1 second' <= statement_timestamp()" - ), - ) - .values(status=JobStatus.TIMED_OUT) - ) - return typing.cast(typing.Any, result).rowcount or 0 - - async def pending(self): - return tuple( - await self._session.scalars( - sqlmodel.select(JobModel).where(JobModel.status == JobStatus.PENDING) - ) - ) +if TYPE_CHECKING: + from inkcre_core.persistence.job.repository import * diff --git a/app/persistence/job/uow.py b/app/persistence/job/uow.py index 11641f3f..10c30c67 100644 --- a/app/persistence/job/uow.py +++ b/app/persistence/job/uow.py @@ -1,21 +1,10 @@ -"""Job admission also reads Source contracts within its transaction.""" +"""Legacy import; use inkcre_core for new consumers.""" -from collections.abc import AsyncGenerator -from contextlib import asynccontextmanager -from dataclasses import dataclass +from typing import TYPE_CHECKING +from importlib import import_module +import sys -from app.engine import AsyncSessionFactory -from app.persistence.source.repository import SourceRepository -from .repository import JobRepository +sys.modules[__name__] = import_module("inkcre_core.persistence.job.uow") - -@dataclass(frozen=True) -class JobUnitOfWork: - jobs: JobRepository - sources: SourceRepository - - -@asynccontextmanager -async def job_uow() -> AsyncGenerator[JobUnitOfWork, None]: - async with AsyncSessionFactory.begin() as session: - yield JobUnitOfWork(JobRepository(session), SourceRepository(session)) +if TYPE_CHECKING: + from inkcre_core.persistence.job.uow import * diff --git a/app/persistence/lexical_retrieval/__init__.py b/app/persistence/lexical_retrieval/__init__.py index 05dca4e7..6eee1fd6 100644 --- a/app/persistence/lexical_retrieval/__init__.py +++ b/app/persistence/lexical_retrieval/__init__.py @@ -1 +1,3 @@ -"""Lexical projection persistence.""" +"""Legacy import; use inkcre_core for new consumers.""" + +from inkcre_core.persistence.lexical_retrieval import * diff --git a/app/persistence/lexical_retrieval/repository.py b/app/persistence/lexical_retrieval/repository.py index 52413a52..d2520dcc 100644 --- a/app/persistence/lexical_retrieval/repository.py +++ b/app/persistence/lexical_retrieval/repository.py @@ -1,145 +1,12 @@ -"""Session-bound lexical ranking, candidate scans and projection batch writes.""" +"""Legacy import; use inkcre_core for new consumers.""" -import datetime -import typing +from typing import TYPE_CHECKING +from importlib import import_module +import sys -import sqlalchemy -from sqlalchemy.dialects.postgresql import insert -from sqlalchemy.ext.asyncio import AsyncSession -import sqlmodel +sys.modules[__name__] = import_module( + "inkcre_core.persistence.lexical_retrieval.repository" +) -from app.schemas.info_base.block import BlockModel -from app.schemas.lexical_retrieval import BlockLexicalRecordModel - - -class LexicalRepository: - def __init__(self, session: AsyncSession) -> None: - self._session = session - - async def search(self, normalized: str, limit: int): - literal_pattern = f"%{self._escape_like(normalized)}%" - - block_columns = typing.cast( - typing.Any, - BlockModel.__table__.c, # pyrefly: ignore[missing-attribute] - ) - record_columns = typing.cast( - typing.Any, - BlockLexicalRecordModel.__table__.c, # pyrefly: ignore[missing-attribute] - ) - query_terms = sqlalchemy.func.plainto_tsquery("simple", normalized) - label_exact = sqlalchemy.func.lower(record_columns.label) == normalized.lower() - label_substring = record_columns.label.ilike(literal_pattern, escape="\\") - text_substring = record_columns.text.ilike(literal_pattern, escape="\\") - term_match = record_columns.search_vector.op("@@")(query_terms) - evidence = sqlalchemy.case( - (label_exact, "label_exact"), - (label_substring, "label_substring"), - (text_substring, "text_substring"), - else_="terms", - ).label("evidence") - evidence_class = sqlalchemy.case( - (label_exact, 4.0), - (label_substring, 3.0), - (text_substring, 2.0), - else_=1.0, - ) - term_rank = sqlalchemy.func.ts_rank_cd(record_columns.search_vector, query_terms) - rank = (evidence_class + term_rank).label("rank") - - statement = ( - sqlmodel.select(BlockModel, BlockLexicalRecordModel, evidence, rank) - .join( - BlockLexicalRecordModel, - record_columns.block == block_columns.id, - ) - .where( - record_columns.updated_at >= block_columns.updated_at, - sqlalchemy.or_( - label_exact, - label_substring, - text_substring, - term_match, - ), - ) - .order_by( - sqlalchemy.desc(evidence_class), - sqlalchemy.desc(term_rank), - block_columns.id, - ) - .limit(limit) - ) - return (await self._session.execute(statement)).all() - - async def candidate_page( - self, - cursor: int, - page_size: int, - rebuild_cutoff: datetime.datetime | None, - ) -> tuple[tuple[BlockModel, ...], int, bool]: - block_columns = typing.cast( - typing.Any, - BlockModel.__table__.c, # pyrefly: ignore[missing-attribute] - ) - record_columns = typing.cast( - typing.Any, - BlockLexicalRecordModel.__table__.c, # pyrefly: ignore[missing-attribute] - ) - needs_projection = sqlalchemy.or_( - record_columns.block.is_(None), - record_columns.updated_at < block_columns.updated_at, - ) - if rebuild_cutoff is not None: - needs_projection = sqlalchemy.or_( - needs_projection, - record_columns.updated_at < rebuild_cutoff, - ) - statement = ( - sqlmodel.select(BlockModel) - .outerjoin( - BlockLexicalRecordModel, - record_columns.block == block_columns.id, - ) - .where(block_columns.id > cursor, needs_projection) - .order_by(block_columns.id) - .limit(page_size) - ) - blocks = tuple(await self._session.scalars(statement)) - next_cursor = typing.cast(int, blocks[-1].id) if blocks else cursor - return blocks, next_cursor, len(blocks) < page_size - - async def database_now(self) -> datetime.datetime: - return ( - await self._session.execute(sqlmodel.select(sqlalchemy.func.current_timestamp())) - ).scalar_one() - - async def upsert(self, projections: list[dict[str, typing.Any]]) -> None: - if not projections: - return - rows = [] - for projection in projections: - label_vector = sqlalchemy.func.setweight( - sqlalchemy.func.to_tsvector("simple", projection["label"]), - sqlalchemy.literal_column("'A'::\"char\""), - ) - text_vector = sqlalchemy.func.setweight( - sqlalchemy.func.to_tsvector("simple", projection["text"] or ""), - sqlalchemy.literal_column("'D'::\"char\""), - ) - rows.append({**projection, "search_vector": label_vector.op("||")(text_vector)}) - statement = insert(BlockLexicalRecordModel).values(rows) - await self._session.execute( - statement.on_conflict_do_update( - index_elements=["block"], - set_={ - "label": statement.excluded.label, - "text": statement.excluded.text, - "search_vector": statement.excluded.search_vector, - "updated_at": sqlalchemy.func.current_timestamp(), - }, - ) - ) - - @staticmethod - def _escape_like(value: str) -> str: - return value.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") +if TYPE_CHECKING: + from inkcre_core.persistence.lexical_retrieval.repository import * diff --git a/app/persistence/lexical_retrieval/uow.py b/app/persistence/lexical_retrieval/uow.py index 3a1338c3..b5ed6aad 100644 --- a/app/persistence/lexical_retrieval/uow.py +++ b/app/persistence/lexical_retrieval/uow.py @@ -1,13 +1,10 @@ -"""Short lexical database scope, separate from Resolver projection.""" +"""Legacy import; use inkcre_core for new consumers.""" -from collections.abc import AsyncGenerator -from contextlib import asynccontextmanager +from typing import TYPE_CHECKING +from importlib import import_module +import sys -from app.engine import AsyncSessionFactory -from .repository import LexicalRepository +sys.modules[__name__] = import_module("inkcre_core.persistence.lexical_retrieval.uow") - -@asynccontextmanager -async def lexical_uow() -> AsyncGenerator[LexicalRepository, None]: - async with AsyncSessionFactory.begin() as session: - yield LexicalRepository(session) +if TYPE_CHECKING: + from inkcre_core.persistence.lexical_retrieval.uow import * diff --git a/app/persistence/observability/__init__.py b/app/persistence/observability/__init__.py index 302d1365..6f9aab43 100644 --- a/app/persistence/observability/__init__.py +++ b/app/persistence/observability/__init__.py @@ -1 +1,3 @@ -"""Isolated asynchronous telemetry persistence.""" +"""Legacy import; use inkcre_core for new consumers.""" + +from inkcre_core.persistence.observability import * diff --git a/app/persistence/observability/uow.py b/app/persistence/observability/uow.py index a5a21f95..fe555525 100644 --- a/app/persistence/observability/uow.py +++ b/app/persistence/observability/uow.py @@ -1,11 +1,10 @@ -"""Log batches commit independently of application transactions.""" +"""Legacy import; use inkcre_core for new consumers.""" -from collections.abc import Sequence +from typing import TYPE_CHECKING +from importlib import import_module +import sys -from app.engine import AsyncSessionFactory -from libs.obsrv.log_record import LogModel +sys.modules[__name__] = import_module("inkcre_core.persistence.observability.uow") - -async def write_logs(entries: Sequence[LogModel]) -> None: - async with AsyncSessionFactory.begin() as session: - session.add_all(entries) +if TYPE_CHECKING: + from inkcre_core.persistence.observability.uow import * diff --git a/app/persistence/peer/__init__.py b/app/persistence/peer/__init__.py index e69de29b..6d1c7c64 100644 --- a/app/persistence/peer/__init__.py +++ b/app/persistence/peer/__init__.py @@ -0,0 +1,3 @@ +"""Legacy import; use inkcre_core for new consumers.""" + +from inkcre_core.persistence.peer import * diff --git a/app/persistence/peer/repository.py b/app/persistence/peer/repository.py index 886d3e7c..e31b9143 100644 --- a/app/persistence/peer/repository.py +++ b/app/persistence/peer/repository.py @@ -1,106 +1,10 @@ -"""Peer persistence; all lease comparisons use the database clock.""" +"""Legacy import; use inkcre_core for new consumers.""" -import datetime -from typing import Any +from typing import TYPE_CHECKING +from importlib import import_module +import sys -import sqlalchemy -from sqlalchemy.dialects.postgresql import insert -from sqlalchemy.ext.asyncio import AsyncSession -import sqlmodel +sys.modules[__name__] = import_module("inkcre_core.persistence.peer.repository") -from app.database_contract import PROTOCOL_SCHEMA -from app.schemas.peer import PeerModel, PeerRef - - -class PeerRepository: - def __init__(self, session: AsyncSession) -> None: - self._session = session - - async def register( - self, - peer_id: PeerRef, - name: str, - application_version: str, - schema: dict, - ) -> PeerModel: - statement = insert(PeerModel).values( - id=peer_id, - name=name, - application_version=application_version, - labels=[], - config={}, - config_schema=schema, - capabilities=[], - ) - statement = statement.on_conflict_do_update( - index_elements=["id"], - set_={ - "application_version": statement.excluded.application_version, - "config_schema": statement.excluded.config_schema, - }, - ).returning(PeerModel) - return (await self._session.scalars(statement)).one() - - async def get(self, peer_id: PeerRef, *, for_update: bool = False) -> PeerModel | None: - statement = sqlmodel.select(PeerModel).where(PeerModel.id == peer_id) - if for_update: - statement = statement.with_for_update() - return (await self._session.scalars(statement)).one_or_none() - - async def get_all(self) -> tuple[PeerModel, ...]: - return tuple((await self._session.scalars(sqlmodel.select(PeerModel))).all()) - - async def with_leases( - self, - *, - peer_id: PeerRef | None = None, - limit: int | None = None, - cursor: PeerRef | None = None, - ) -> tuple[list[dict[str, Any]], PeerRef | None]: - statement = sqlmodel.select( - PeerModel, - sqlalchemy.func.coalesce( - PeerModel.lease_expires_at > sqlalchemy.func.statement_timestamp(), False - ), - ).order_by(sqlmodel.col(PeerModel.id)) - if peer_id is not None: - statement = statement.where(PeerModel.id == peer_id) - if cursor is not None: - statement = statement.where(PeerModel.id > cursor) - if limit is not None: - statement = statement.limit(limit + 1) - rows = list((await self._session.execute(statement)).all()) - more = limit is not None and len(rows) > limit - rows = rows[:limit] - return [ - {**row.model_dump(mode="json"), "lease_active": active} for row, active in rows - ], rows[-1][0].id if more else None - - async def save(self, peer: PeerModel) -> None: - self._session.add(peer) - await self._session.flush() - await self._session.refresh(peer) - - async def renew_lease(self, peer_id: PeerRef, ttl_seconds: int) -> datetime.datetime: - statement = sqlalchemy.select( - getattr(sqlalchemy.func, PROTOCOL_SCHEMA).renew_peer_lease(peer_id, ttl_seconds) - ) - return (await self._session.execute(statement)).scalar_one() - - async def clear_lease(self, peer_id: PeerRef) -> None: - await self._session.execute( - sqlalchemy.update(PeerModel) - .where(sqlmodel.col(PeerModel.id) == peer_id) - .values(lease_expires_at=None) - ) - - async def candidates( - self, self_id: PeerRef, route_to_peer: PeerRef | None - ) -> tuple[PeerModel, ...]: - statement = sqlmodel.select(PeerModel).where( - PeerModel.id != self_id, - PeerModel.lease_expires_at > sqlalchemy.func.statement_timestamp(), - ) - if route_to_peer is not None: - statement = statement.where(PeerModel.id == route_to_peer) - return tuple((await self._session.scalars(statement)).all()) +if TYPE_CHECKING: + from inkcre_core.persistence.peer.repository import * diff --git a/app/persistence/peer/uow.py b/app/persistence/peer/uow.py index 3a18b553..9e0780bb 100644 --- a/app/persistence/peer/uow.py +++ b/app/persistence/peer/uow.py @@ -1,13 +1,10 @@ -"""Transaction scopes for Peer facts and database-owned leases.""" +"""Legacy import; use inkcre_core for new consumers.""" -from collections.abc import AsyncGenerator -from contextlib import asynccontextmanager +from typing import TYPE_CHECKING +from importlib import import_module +import sys -from app.engine import AsyncSessionFactory -from .repository import PeerRepository +sys.modules[__name__] = import_module("inkcre_core.persistence.peer.uow") - -@asynccontextmanager -async def peer_uow() -> AsyncGenerator[PeerRepository, None]: - async with AsyncSessionFactory.begin() as session: - yield PeerRepository(session) +if TYPE_CHECKING: + from inkcre_core.persistence.peer.uow import * diff --git a/app/persistence/semantic_retrieval/__init__.py b/app/persistence/semantic_retrieval/__init__.py index d396cf8b..8df71f07 100644 --- a/app/persistence/semantic_retrieval/__init__.py +++ b/app/persistence/semantic_retrieval/__init__.py @@ -1 +1,3 @@ -"""Semantic retrieval persistence.""" +"""Legacy import; use inkcre_core for new consumers.""" + +from inkcre_core.persistence.semantic_retrieval import * diff --git a/app/persistence/semantic_retrieval/repository.py b/app/persistence/semantic_retrieval/repository.py index e2619d92..fc3d23db 100644 --- a/app/persistence/semantic_retrieval/repository.py +++ b/app/persistence/semantic_retrieval/repository.py @@ -1,203 +1,12 @@ -"""Session-bound embedding profiles and derived retrieval records.""" +"""Legacy import; use inkcre_core for new consumers.""" -import datetime -import typing +from typing import TYPE_CHECKING +from importlib import import_module +import sys -import sqlalchemy -import sqlalchemy.orm -from sqlalchemy.dialects.postgresql import insert -from sqlalchemy.ext.asyncio import AsyncSession -import sqlmodel - -from app.schemas.info_base.block import BlockModel -from app.schemas.info_base.relation import RelationModel -from app.schemas.semantic_retrieval import VectorRetrievalOptions - -from app.schemas.ai import ( - EmbeddingProfileID, - EmbeddingProfileModel, - BlockEmbeddingModel, - RelationEmbeddingModel, +sys.modules[__name__] = import_module( + "inkcre_core.persistence.semantic_retrieval.repository" ) - -class SemanticRetrievalRepository: - def __init__(self, session: AsyncSession) -> None: - self._session = session - - async def get_profile( - self, profile_id: EmbeddingProfileID - ) -> EmbeddingProfileModel | None: - return await self._session.get(EmbeddingProfileModel, profile_id) - - async def list_profiles(self, *, limit: int | None, cursor: int | None): - statement = sqlmodel.select(EmbeddingProfileModel).order_by( - sqlmodel.col(EmbeddingProfileModel.id) - ) - if cursor is not None: - statement = statement.where(sqlmodel.col(EmbeddingProfileModel.id) > cursor) - if limit is not None: - statement = statement.limit(limit + 1) - rows = list(await self._session.scalars(statement)) - more = limit is not None and len(rows) > limit - rows = rows[:limit] - return rows, rows[-1].id if more else None - - async def database_now(self) -> datetime.datetime: - return ( - await self._session.execute(sqlmodel.select(sqlalchemy.func.current_timestamp())) - ).scalar_one() - - async def candidate_blocks(self, profile_id: int, cursor: int, page_size: int): - block_columns = typing.cast( - typing.Any, - BlockModel.__table__.c, # pyrefly: ignore[missing-attribute] - ) - record_columns = typing.cast( - typing.Any, - BlockEmbeddingModel.__table__.c, # pyrefly: ignore[missing-attribute] - ) - statement = ( - sqlmodel.select(BlockModel, BlockEmbeddingModel) - .outerjoin( - BlockEmbeddingModel, - sqlalchemy.and_( - record_columns.profile == profile_id, - record_columns.block == block_columns.id, - ), - ) - .where(block_columns.id > cursor) - .order_by(block_columns.id) - .limit(page_size) - ) - return (await self._session.execute(statement)).all() - - async def candidate_relations(self, profile_id: int, cursor: int, page_size: int): - from_block = sqlalchemy.orm.aliased(BlockModel) - to_block = sqlalchemy.orm.aliased(BlockModel) - from_columns = typing.cast(typing.Any, from_block) - to_columns = typing.cast(typing.Any, to_block) - relation_columns = typing.cast( - typing.Any, - RelationModel.__table__.c, # pyrefly: ignore[missing-attribute] - ) - record_columns = typing.cast( - typing.Any, - RelationEmbeddingModel.__table__.c, # pyrefly: ignore[missing-attribute] - ) - statement = ( - sqlmodel.select( - RelationModel, - RelationEmbeddingModel, - from_block.updated_at, - to_block.updated_at, - ) - .outerjoin( - RelationEmbeddingModel, - sqlalchemy.and_( - record_columns.profile == profile_id, - record_columns.relation == relation_columns.id, - ), - ) - .join(from_block, from_columns.id == relation_columns.from_) - .join(to_block, to_columns.id == relation_columns.to_) - .where(relation_columns.id > cursor) - .order_by(relation_columns.id) - .limit(page_size) - ) - return (await self._session.execute(statement)).all() - - async def retrieve_blocks( - self, - profile: EmbeddingProfileModel, - query_vector, - options: VectorRetrievalOptions, - ): - profile_id = profile.id - block_columns = typing.cast( - typing.Any, - BlockModel.__table__.c, # pyrefly: ignore[missing-attribute] - ) - record_columns = typing.cast( - typing.Any, - BlockEmbeddingModel.__table__.c, # pyrefly: ignore[missing-attribute] - ) - distance = record_columns.embedding.cosine_distance(query_vector).label("distance") - statement = ( - sqlmodel.select(BlockModel, distance) - .join(BlockEmbeddingModel, record_columns.block == block_columns.id) - .where( - record_columns.profile == profile_id, - record_columns.updated_at >= profile.updated_at, - record_columns.updated_at >= block_columns.updated_at, - sqlalchemy.func.vector_dims(record_columns.embedding) == profile.dimensions, - sqlalchemy.func.vector_norm(record_columns.embedding) > 0, - ) - .order_by(distance, block_columns.id) - .limit(options.limit) - ) - if options.min_score is not None: - statement = statement.where(distance <= 1 - options.min_score) - return (await self._session.execute(statement)).all() - - async def retrieve_relations( - self, - profile: EmbeddingProfileModel, - query_vector, - options: VectorRetrievalOptions, - ): - profile_id = profile.id - from_block = sqlalchemy.orm.aliased(BlockModel) - to_block = sqlalchemy.orm.aliased(BlockModel) - from_columns = typing.cast(typing.Any, from_block) - to_columns = typing.cast(typing.Any, to_block) - relation_columns = typing.cast( - typing.Any, - RelationModel.__table__.c, # pyrefly: ignore[missing-attribute] - ) - record_columns = typing.cast( - typing.Any, - RelationEmbeddingModel.__table__.c, # pyrefly: ignore[missing-attribute] - ) - distance = record_columns.embedding.cosine_distance(query_vector).label("distance") - statement = ( - sqlmodel.select(RelationModel, distance) - .join( - RelationEmbeddingModel, - record_columns.relation == relation_columns.id, - ) - .join(from_block, from_columns.id == relation_columns.from_) - .join(to_block, to_columns.id == relation_columns.to_) - .where( - record_columns.profile == profile_id, - record_columns.updated_at >= profile.updated_at, - record_columns.updated_at >= relation_columns.updated_at, - record_columns.updated_at >= from_columns.updated_at, - record_columns.updated_at >= to_columns.updated_at, - sqlalchemy.func.vector_dims(record_columns.embedding) == profile.dimensions, - sqlalchemy.func.vector_norm(record_columns.embedding) > 0, - ) - .order_by(distance, relation_columns.id) - .limit(options.limit) - ) - if options.min_score is not None: - statement = statement.where(distance <= 1 - options.min_score) - return (await self._session.execute(statement)).all() - - async def upsert(self, blocks: list[dict], relations: list[dict]) -> None: - for model, key, rows in ( - (BlockEmbeddingModel, "block", blocks), - (RelationEmbeddingModel, "relation", relations), - ): - if not rows: - continue - statement = insert(model).values(rows) - await self._session.execute( - statement.on_conflict_do_update( - index_elements=["profile", key], - set_={ - "embedding": statement.excluded.embedding, - "updated_at": sqlalchemy.func.current_timestamp(), - }, - ) - ) +if TYPE_CHECKING: + from inkcre_core.persistence.semantic_retrieval.repository import * diff --git a/app/persistence/semantic_retrieval/uow.py b/app/persistence/semantic_retrieval/uow.py index 81e2f460..5021ea29 100644 --- a/app/persistence/semantic_retrieval/uow.py +++ b/app/persistence/semantic_retrieval/uow.py @@ -1,13 +1,10 @@ -"""Short persistence scope; provider calls run after it closes.""" +"""Legacy import; use inkcre_core for new consumers.""" -from collections.abc import AsyncGenerator -from contextlib import asynccontextmanager +from typing import TYPE_CHECKING +from importlib import import_module +import sys -from app.engine import AsyncSessionFactory -from .repository import SemanticRetrievalRepository +sys.modules[__name__] = import_module("inkcre_core.persistence.semantic_retrieval.uow") - -@asynccontextmanager -async def semantic_retrieval_uow() -> AsyncGenerator[SemanticRetrievalRepository, None]: - async with AsyncSessionFactory.begin() as session: - yield SemanticRetrievalRepository(session) +if TYPE_CHECKING: + from inkcre_core.persistence.semantic_retrieval.uow import * diff --git a/app/persistence/sink/__init__.py b/app/persistence/sink/__init__.py index e69de29b..d54f3682 100644 --- a/app/persistence/sink/__init__.py +++ b/app/persistence/sink/__init__.py @@ -0,0 +1,3 @@ +"""Legacy import; use inkcre_core for new consumers.""" + +from inkcre_core.persistence.sink import * diff --git a/app/persistence/sink/repository.py b/app/persistence/sink/repository.py index 52cb3f11..3586e188 100644 --- a/app/persistence/sink/repository.py +++ b/app/persistence/sink/repository.py @@ -1,55 +1,10 @@ -"""Session-bound Sink catalog and instance records.""" +"""Legacy import; use inkcre_core for new consumers.""" -import sqlmodel -from sqlalchemy.dialects.postgresql import insert -from sqlalchemy.ext.asyncio import AsyncSession +from typing import TYPE_CHECKING +from importlib import import_module +import sys -from app.schemas.sink import SinkModel, SinkTypeModel +sys.modules[__name__] = import_module("inkcre_core.persistence.sink.repository") - -class SinkRepository: - def __init__(self, session: AsyncSession) -> None: - self._session = session - - async def sync_types(self, rows: list[dict]) -> None: - if not rows: - return - statement = insert(SinkTypeModel).values(rows) - await self._session.execute( - statement.on_conflict_do_update( - index_elements=["id"], - set_={ - "description": statement.excluded.description, - "config_schema": statement.excluded.config_schema, - }, - ) - ) - - async def list_types(self): - return tuple( - await self._session.scalars(sqlmodel.select(SinkTypeModel).order_by(SinkTypeModel.id)) - ) - - async def get_type(self, sink_type: str) -> SinkTypeModel | None: - return await self._session.get(SinkTypeModel, sink_type) - - async def list(self): - return tuple( - await self._session.scalars( - sqlmodel.select(SinkModel).order_by(sqlmodel.col(SinkModel.id)) - ) - ) - - async def get(self, sink_id: int, *, lock: bool = False) -> SinkModel | None: - statement = sqlmodel.select(SinkModel).where(SinkModel.id == sink_id) - if lock: - statement = statement.with_for_update() - return (await self._session.scalars(statement)).one_or_none() - - async def save(self, sink: SinkModel) -> None: - self._session.add(sink) - await self._session.flush() - await self._session.refresh(sink) - - async def delete(self, sink: SinkModel) -> None: - await self._session.delete(sink) +if TYPE_CHECKING: + from inkcre_core.persistence.sink.repository import * diff --git a/app/persistence/sink/uow.py b/app/persistence/sink/uow.py index b6128d8f..2469d6c7 100644 --- a/app/persistence/sink/uow.py +++ b/app/persistence/sink/uow.py @@ -1,13 +1,10 @@ -"""Short Sink persistence scope; runtime start/close happen after commit.""" +"""Legacy import; use inkcre_core for new consumers.""" -from collections.abc import AsyncGenerator -from contextlib import asynccontextmanager +from typing import TYPE_CHECKING +from importlib import import_module +import sys -from app.engine import AsyncSessionFactory -from .repository import SinkRepository +sys.modules[__name__] = import_module("inkcre_core.persistence.sink.uow") - -@asynccontextmanager -async def sink_uow() -> AsyncGenerator[SinkRepository, None]: - async with AsyncSessionFactory.begin() as session: - yield SinkRepository(session) +if TYPE_CHECKING: + from inkcre_core.persistence.sink.uow import * diff --git a/app/persistence/source/__init__.py b/app/persistence/source/__init__.py index e69de29b..f57c089a 100644 --- a/app/persistence/source/__init__.py +++ b/app/persistence/source/__init__.py @@ -0,0 +1,3 @@ +"""Legacy import; use inkcre_core for new consumers.""" + +from inkcre_core.persistence.source import * diff --git a/app/persistence/source/repository.py b/app/persistence/source/repository.py index 2fd9d1ec..04992082 100644 --- a/app/persistence/source/repository.py +++ b/app/persistence/source/repository.py @@ -1,76 +1,10 @@ -"""Persistence for Source catalog and instances.""" +"""Legacy import; use inkcre_core for new consumers.""" -import sqlmodel -from sqlalchemy.dialects.postgresql import insert -from sqlalchemy.ext.asyncio import AsyncSession +from typing import TYPE_CHECKING +from importlib import import_module +import sys -from app.schemas.source import SourceModel, SourceTypesModel +sys.modules[__name__] = import_module("inkcre_core.persistence.source.repository") - -class SourceRepository: - def __init__(self, session: AsyncSession) -> None: - self._session = session - - async def sync_types(self, rows: list[dict]) -> None: - if not rows: - return - statement = insert(SourceTypesModel).values(rows) - statement = statement.on_conflict_do_update( - index_elements=[SourceTypesModel.id], - set_={ - key: getattr(statement.excluded, key) - for key in ( - "description", - "config_schema", - "collect_config_schema", - "backfill_config_schema", - ) - }, - ) - await self._session.execute(statement) - - async def get(self, source_id: int, *, lock: bool = False): - statement = sqlmodel.select(SourceModel).where(SourceModel.id == source_id) - if lock: - statement = statement.with_for_update() - return (await self._session.scalars(statement)).one_or_none() - - async def get_by_block(self, block_id: int) -> SourceModel | None: - return ( - await self._session.scalars( - sqlmodel.select(SourceModel).where(SourceModel.block == block_id) - ) - ).one_or_none() - - async def save(self, source: SourceModel) -> None: - self._session.add(source) - await self._session.flush() - await self._session.refresh(source) - - async def get_type(self, type_: str) -> SourceTypesModel | None: - return await self._session.get(SourceTypesModel, type_) - - async def list_sources(self, *, limit: int | None, cursor: int | None): - statement = sqlmodel.select(SourceModel).order_by(sqlmodel.col(SourceModel.id)) - if cursor is not None: - statement = statement.where(sqlmodel.col(SourceModel.id) > cursor) - if limit is not None: - statement = statement.limit(limit + 1) - rows = list((await self._session.scalars(statement)).all()) - more = limit is not None and len(rows) > limit - rows = rows[:limit] - return rows, rows[-1].id if more else None - - async def list_types(self, *, limit: int | None, cursor: str | None): - statement = sqlmodel.select(SourceTypesModel).order_by(SourceTypesModel.id) - if cursor is not None: - statement = statement.where(SourceTypesModel.id > cursor) - if limit is not None: - statement = statement.limit(limit + 1) - rows = list((await self._session.scalars(statement)).all()) - more = limit is not None and len(rows) > limit - rows = rows[:limit] - return rows, rows[-1].id if more else None - - async def delete(self, source: SourceModel) -> None: - await self._session.delete(source) +if TYPE_CHECKING: + from inkcre_core.persistence.source.repository import * diff --git a/app/persistence/source/uow.py b/app/persistence/source/uow.py index 3e3b13bf..ec520a9d 100644 --- a/app/persistence/source/uow.py +++ b/app/persistence/source/uow.py @@ -1,31 +1,10 @@ -"""Source state, graph and storage participate in one short transaction.""" +"""Legacy import; use inkcre_core for new consumers.""" -from collections.abc import AsyncGenerator -from contextlib import asynccontextmanager -from dataclasses import dataclass +from typing import TYPE_CHECKING +from importlib import import_module +import sys -from app.engine import AsyncSessionFactory -from app.persistence.info_base.uow import GraphUnitOfWork -from app.persistence.info_base.repository import BlockRepository, RelationRepository -from app.persistence.info_base.storage import StorageRepository -from app.persistence.deployment_config.repository import DeploymentConfigRepository -from .repository import SourceRepository +sys.modules[__name__] = import_module("inkcre_core.persistence.source.uow") - -@dataclass(frozen=True) -class SourceUnitOfWork: - sources: SourceRepository - graph: GraphUnitOfWork - configuration: DeploymentConfigRepository - - -@asynccontextmanager -async def source_uow() -> AsyncGenerator[SourceUnitOfWork, None]: - async with AsyncSessionFactory.begin() as session: - yield SourceUnitOfWork( - SourceRepository(session), - GraphUnitOfWork( - BlockRepository(session), RelationRepository(session), StorageRepository(session) - ), - DeploymentConfigRepository(session), - ) +if TYPE_CHECKING: + from inkcre_core.persistence.source.uow import * diff --git a/app/routes/agent.py b/app/routes/agent.py index 18dcb15a..002f56c8 100644 --- a/app/routes/agent.py +++ b/app/routes/agent.py @@ -4,10 +4,10 @@ import fastapi -from app.business.agent.main import AgentManager -from app.business.agent.contracts import AgentNotFoundError, MissingAgentToolError -from app.schemas.agent import AgentDefinitionModel, AgentForm, AgentUpdateForm -from app.schemas.ai import FunctionTool +from inkcre_core.business.agent.main import AgentManager +from inkcre_core.business.agent.contracts import AgentNotFoundError, MissingAgentToolError +from inkcre_core.schemas.agent import AgentDefinitionModel, AgentForm, AgentUpdateForm +from inkcre_core.schemas.ai import FunctionTool from .validation import database_write, request_input diff --git a/app/routes/ai.py b/app/routes/ai.py index 1faa7882..1b17fb02 100644 --- a/app/routes/ai.py +++ b/app/routes/ai.py @@ -4,8 +4,8 @@ import fastapi -from app.business.ai import AIManager -from app.schemas.ai import AIModelModel +from inkcre_core.business.ai import AIManager +from inkcre_core.schemas.ai import AIModelModel ROUTER = fastapi.APIRouter(prefix="/ai", tags=["ai"]) diff --git a/app/routes/block.py b/app/routes/block.py index 43c2f139..c63ba46f 100644 --- a/app/routes/block.py +++ b/app/routes/block.py @@ -3,10 +3,10 @@ __all__ = ["ROUTER"] import fastapi -from app.business.info_base.services import BlockService -from app.schemas.info_base.block import BlockModel -from app.schemas.info_base.block import BlockForm -from app.schemas.info_base.rest import BlockUpdateForm +from inkcre_core.business.info_base.services import BlockService +from inkcre_core.schemas.info_base.block import BlockModel +from inkcre_core.schemas.info_base.block import BlockForm +from inkcre_core.schemas.info_base.rest import BlockUpdateForm from .validation import database_write ROUTER = fastapi.APIRouter( diff --git a/app/routes/cron.py b/app/routes/cron.py index ab498eea..44b3553c 100644 --- a/app/routes/cron.py +++ b/app/routes/cron.py @@ -5,11 +5,18 @@ import fastapi -from app.business.cron import CronManager, CronNotFoundError, InvalidCronScheduleError -from app.business.job import JobManager, UnknownJobTypeError -from app.business.source.main import SourceNotFoundError, UnsupportedSourceCommandError -from app.schemas.cron import CronForm, CronModel, CronUpdateForm -from app.schemas.job import JobModel +from inkcre_core.business.cron import ( + CronManager, + CronNotFoundError, + InvalidCronScheduleError, +) +from inkcre_core.business.job import JobManager, UnknownJobTypeError +from inkcre_core.business.source.main import ( + SourceNotFoundError, + UnsupportedSourceCommandError, +) +from inkcre_core.schemas.cron import CronForm, CronModel, CronUpdateForm +from inkcre_core.schemas.job import JobModel from .validation import request_input diff --git a/app/routes/deployment_config.py b/app/routes/deployment_config.py index 327d6733..0e32273f 100644 --- a/app/routes/deployment_config.py +++ b/app/routes/deployment_config.py @@ -6,13 +6,13 @@ import fastapi -from app.business.deployment_config import ( +from inkcre_core.business.deployment_config import ( DeploymentConfigManager, DeploymentConfigService, DeploymentConfigNotFoundError, UnknownDeploymentConfigSchemaError, ) -from app.schemas.deployment_config import ( +from inkcre_core.schemas.deployment_config import ( DeploymentConfigKey, DeploymentConfigReplaceForm, DeploymentConfigView, diff --git a/app/routes/entities.py b/app/routes/entities.py index e3882f68..416019d4 100644 --- a/app/routes/entities.py +++ b/app/routes/entities.py @@ -4,10 +4,10 @@ import fastapi -from app.business.info_base import get_entity_records -from app.schemas.info_base.relation import RelationModel -from app.schemas.info_base.rest import EntitiesGetForm -from libs.obsrv.main import get_logger +from inkcre_core.business.info_base import get_entity_records +from inkcre_core.schemas.info_base.relation import RelationModel +from inkcre_core.schemas.info_base.rest import EntitiesGetForm +from inkcre_core.libs.obsrv.main import get_logger from .content import CONTENT_RESPONSES, content_response diff --git a/app/routes/extension.py b/app/routes/extension.py index efc71c2e..31e9e17a 100644 --- a/app/routes/extension.py +++ b/app/routes/extension.py @@ -11,8 +11,8 @@ InstalledExtension, ) from app.business.extension.errors import ExtensionHostError -from app.business.peer import PeerHTTPInbound, PeerManager -from app.schemas.extension import ( +from inkcre_core.business.peer import PeerHTTPInbound, PeerManager +from inkcre_core.schemas.extension import ( DisableExtensionCommand, EnableExtensionCommand, ExtensionManagementCommand, @@ -21,7 +21,7 @@ SelectExtensionInput, UnselectExtensionCommand, ) -from app.schemas.peer import PeerRef +from inkcre_core.schemas.peer import PeerRef from .validation import request_input diff --git a/app/routes/graph_navigation.py b/app/routes/graph_navigation.py index c182b409..233738e9 100644 --- a/app/routes/graph_navigation.py +++ b/app/routes/graph_navigation.py @@ -3,14 +3,14 @@ import fastapi import pydantic -from app.business.graph_navigation_retrieval import GraphNavigationRetrievalManager -from app.schemas.graph_navigation_retrieval import ( +from inkcre_core.business.graph_navigation_retrieval import GraphNavigationRetrievalManager +from inkcre_core.schemas.graph_navigation_retrieval import ( GraphDirection, GraphModel, DEFAULT_NEIGHBORHOOD_LIMIT, MAX_NEIGHBORHOOD_LIMIT, ) -from app.schemas.info_base.rest import GraphPathForm, GraphComponentsForm +from inkcre_core.schemas.info_base.rest import GraphPathForm, GraphComponentsForm from .entities import relation_record diff --git a/app/routes/info_base.py b/app/routes/info_base.py index aea2abcb..6af39b08 100644 --- a/app/routes/info_base.py +++ b/app/routes/info_base.py @@ -4,8 +4,8 @@ import fastapi -from app.business.info_base.commands import submit_graph as submit_graph_command -from app.schemas.info_base.main import GraphForm, SubmitGraphResult +from inkcre_core.business.info_base.commands import submit_graph as submit_graph_command +from inkcre_core.schemas.info_base.main import GraphForm, SubmitGraphResult from .validation import database_write diff --git a/app/routes/job.py b/app/routes/job.py index 55d6fddd..01f0db11 100644 --- a/app/routes/job.py +++ b/app/routes/job.py @@ -4,10 +4,13 @@ import fastapi -from app.business.job import JobManager, UnknownJobTypeError -from app.business.source.main import SourceNotFoundError, UnsupportedSourceCommandError +from inkcre_core.business.job import JobManager, UnknownJobTypeError +from inkcre_core.business.source.main import ( + SourceNotFoundError, + UnsupportedSourceCommandError, +) from app.business.sink import SinkError, SinkNotFoundError -from app.schemas.job import JobCreateForm, JobModel, JobStatus, JobTypeModel +from inkcre_core.schemas.job import JobCreateForm, JobModel, JobStatus, JobTypeModel from .validation import request_input diff --git a/app/routes/lexical_retrieval.py b/app/routes/lexical_retrieval.py index 5d541852..23bf6e5a 100644 --- a/app/routes/lexical_retrieval.py +++ b/app/routes/lexical_retrieval.py @@ -4,12 +4,12 @@ import fastapi -from app.business.lexical_retrieval import ( +from inkcre_core.business.lexical_retrieval import ( LEXICAL_RETRIEVAL_CAPABILITY, LexicalRetrievalManager, ) -from app.business.peer import PeerHTTPInbound -from app.schemas.lexical_retrieval import ( +from inkcre_core.business.peer import PeerHTTPInbound +from inkcre_core.schemas.lexical_retrieval import ( LexicalRetrievalRequest, LexicalRetrievalResult, ) diff --git a/app/routes/organization.py b/app/routes/organization.py index 903fe0ec..84e459f9 100644 --- a/app/routes/organization.py +++ b/app/routes/organization.py @@ -4,9 +4,12 @@ import fastapi -from app.business.organization import RUMINATION_CAPABILITY, RuminationBehaviorResolver -from app.business.peer import PeerHTTPInbound -from app.schemas.organization import RuminationRequest +from inkcre_core.business.organization import ( + RUMINATION_CAPABILITY, + RuminationBehaviorResolver, +) +from inkcre_core.business.peer import PeerHTTPInbound +from inkcre_core.schemas.organization import RuminationRequest ROUTER = fastapi.APIRouter(tags=["organization"]) diff --git a/app/routes/peer.py b/app/routes/peer.py index cf844771..8fe03988 100644 --- a/app/routes/peer.py +++ b/app/routes/peer.py @@ -4,8 +4,8 @@ import fastapi -from app.business.peer import PeerManager -from app.schemas.peer import PeerRef +from inkcre_core.business.peer import PeerManager +from inkcre_core.schemas.peer import PeerRef ROUTER = fastapi.APIRouter(prefix="/peers", tags=["peer"]) diff --git a/app/routes/relation.py b/app/routes/relation.py index f5f888d7..82dfdecb 100644 --- a/app/routes/relation.py +++ b/app/routes/relation.py @@ -5,9 +5,9 @@ import fastapi -from app.business.info_base.services import RelationService -from app.schemas.info_base.relation import RelationCreateForm -from app.schemas.info_base.rest import RelationUpdateForm +from inkcre_core.business.info_base.services import RelationService +from inkcre_core.schemas.info_base.relation import RelationCreateForm +from inkcre_core.schemas.info_base.rest import RelationUpdateForm from .entities import relation_record from .validation import database_write diff --git a/app/routes/resolver.py b/app/routes/resolver.py index 816966af..224fe077 100644 --- a/app/routes/resolver.py +++ b/app/routes/resolver.py @@ -5,8 +5,8 @@ import fastapi from fastapi.exceptions import RequestValidationError -from app.business.info_base.services import BlockService -from app.business.info_base.resolver import ( +from inkcre_core.business.info_base.services import BlockService +from inkcre_core.business.info_base.resolver import ( ResolverManager, ResolverMethodInputError, UnknownResolverError, diff --git a/app/routes/retrieval.py b/app/routes/retrieval.py index ec511744..7574c928 100644 --- a/app/routes/retrieval.py +++ b/app/routes/retrieval.py @@ -4,16 +4,19 @@ import fastapi -from app.business.lexical_retrieval import LexicalRetrievalManager -from app.business.semantic_retrieval import ( +from inkcre_core.business.lexical_retrieval import LexicalRetrievalManager +from inkcre_core.business.semantic_retrieval import ( SemanticRetrievalManager, SemanticRetrievalNotConfiguredError, EmbeddingProfileNotFoundError, ) -from app.schemas.ai import EmbeddingProfileModel -from app.schemas.peer import PeerRef -from app.schemas.lexical_retrieval import LexicalRetrievalRequest, LexicalRetrievalResult -from app.schemas.semantic_retrieval import SemanticRetrievalRequest +from inkcre_core.schemas.ai import EmbeddingProfileModel +from inkcre_core.schemas.peer import PeerRef +from inkcre_core.schemas.lexical_retrieval import ( + LexicalRetrievalRequest, + LexicalRetrievalResult, +) +from inkcre_core.schemas.semantic_retrieval import SemanticRetrievalRequest from .entities import relation_record diff --git a/app/routes/semantic_retrieval.py b/app/routes/semantic_retrieval.py index 8241bcc6..d56eea67 100644 --- a/app/routes/semantic_retrieval.py +++ b/app/routes/semantic_retrieval.py @@ -4,12 +4,12 @@ import fastapi -from app.business.peer import PeerHTTPInbound -from app.business.semantic_retrieval import ( +from inkcre_core.business.peer import PeerHTTPInbound +from inkcre_core.business.semantic_retrieval import ( SEMANTIC_RETRIEVAL_CAPABILITY, SemanticRetrievalManager, ) -from app.schemas.semantic_retrieval import ( +from inkcre_core.schemas.semantic_retrieval import ( SemanticRetrievalRequest, SemanticRetrievalResult, ) diff --git a/app/routes/sink.py b/app/routes/sink.py index 6dc570d2..af25cc7f 100644 --- a/app/routes/sink.py +++ b/app/routes/sink.py @@ -5,7 +5,7 @@ import fastapi import pydantic -from app.business.peer import PeerManager +from inkcre_core.business.peer import PeerManager from app.business.sink import ( SinkError, SinkManager, @@ -13,7 +13,7 @@ SinkStateConflictError, UnknownSinkTypeError, ) -from app.schemas.sink import SinkCreateForm, SinkID, SinkModel, SinkTypeModel +from inkcre_core.schemas.sink import SinkCreateForm, SinkID, SinkModel, SinkTypeModel ROUTER = fastapi.APIRouter(tags=["sink"]) diff --git a/app/routes/source.py b/app/routes/source.py index cb19a94b..ced8e0b1 100644 --- a/app/routes/source.py +++ b/app/routes/source.py @@ -4,15 +4,15 @@ import fastapi -from app.business.job import JobManager -from app.business.source import SOURCE_BACKFILL_JOB_TYPE, SOURCE_COLLECT_JOB_TYPE -from app.business.source.main import ( +from inkcre_core.business.job import JobManager +from inkcre_core.business.source import SOURCE_BACKFILL_JOB_TYPE, SOURCE_COLLECT_JOB_TYPE +from inkcre_core.business.source.main import ( SourceManager, SourceNotFoundError, UnsupportedSourceCommandError, ) -from app.schemas.job import JobModel -from app.schemas.source import ( +from inkcre_core.schemas.job import JobModel +from inkcre_core.schemas.source import ( SourceCreateForm, SourceModel, SourceTypesModel, diff --git a/app/routes/telemetry.py b/app/routes/telemetry.py index a80e5b61..535e6d6f 100644 --- a/app/routes/telemetry.py +++ b/app/routes/telemetry.py @@ -19,7 +19,7 @@ ExportTraceServiceResponse, ) -from libs.obsrv.telemetry import get_export_connection +from inkcre_core.libs.obsrv.telemetry import get_export_connection ROUTER = fastapi.APIRouter(prefix="/telemetry", tags=["telemetry"]) diff --git a/app/scheduler.py b/app/scheduler.py index 9b5db306..716e6029 100644 --- a/app/scheduler.py +++ b/app/scheduler.py @@ -1,10 +1,11 @@ __all__ = ["scheduler", "with_trace_id"] import asyncio +from app.core import CORE import apscheduler.schedulers.asyncio -from libs.obsrv.log_record import ENABLE_LOG_BACKEND, TRACE_ID +from inkcre_core.libs.obsrv.log_record import ENABLE_LOG_BACKEND, TRACE_ID scheduler = apscheduler.schedulers.asyncio.AsyncIOScheduler() @@ -42,7 +43,8 @@ async def wrapper(*args, **kwargs): token = TRACE_ID.set(trace_id) token_backend = ENABLE_LOG_BACKEND.set(enable_backend) try: - return await coro(*args, **kwargs) + with CORE.bind(): + return await coro(*args, **kwargs) finally: _running.discard(task) TRACE_ID.reset(token) diff --git a/app/schemas/__init__.py b/app/schemas/__init__.py index 0e79a007..0ca92c8e 100644 --- a/app/schemas/__init__.py +++ b/app/schemas/__init__.py @@ -1,56 +1,3 @@ -__all__ = [ - "Base", - "BlockModel", - "StorageModel", - "StorageBlobModel", - "StorageTypesModel", - "RelationModel", - "SourceModel", - "JobTypeModel", - "JobModel", - "CronModel", - "ExtensionModel", - "ExtensionPeerParticipationModel", - "AIDialectModel", - "AIProviderModel", - "AIModelModel", - "EmbeddingProfileModel", - "RelationEmbeddingModel", - "BlockEmbeddingModel", - "BlockLexicalRecordModel", - "AgentDefinitionModel", - "PeerModel", - "DeploymentConfigModel", - "SinkModel", - "SinkTypeModel", -] +"""Legacy import; use inkcre_core for new consumers.""" -import sqlalchemy.orm -import sqlmodel - -from app.database_contract import PROTOCOL_SCHEMA - - -Base = sqlalchemy.orm.declarative_base(metadata=sqlalchemy.MetaData(schema=PROTOCOL_SCHEMA)) -sqlmodel.SQLModel.metadata = Base.metadata - -from .info_base.block import BlockModel -from .info_base.storage import StorageBlobModel, StorageModel, StorageTypesModel -from .info_base.relation import RelationModel -from .source import SourceModel -from .job import JobModel, JobTypeModel -from .cron import CronModel -from .extension.main import ExtensionModel, ExtensionPeerParticipationModel -from .ai import ( - AIDialectModel, - AIModelModel, - AIProviderModel, - BlockEmbeddingModel, - EmbeddingProfileModel, - RelationEmbeddingModel, -) -from .agent import AgentDefinitionModel -from .peer import PeerModel -from .deployment_config import DeploymentConfigModel -from .lexical_retrieval import BlockLexicalRecordModel -from .sink import SinkModel, SinkTypeModel +from inkcre_core.schemas import * diff --git a/app/schemas/agent.py b/app/schemas/agent.py index 5151700d..c9ba0054 100644 --- a/app/schemas/agent.py +++ b/app/schemas/agent.py @@ -1,173 +1,10 @@ -"""Reusable Agent definitions persisted as shared deployment facts.""" +"""Legacy import; use inkcre_core for new consumers.""" -import datetime -import typing +from typing import TYPE_CHECKING +from importlib import import_module +import sys -import pydantic -import sqlalchemy -import sqlalchemy.dialects.postgresql -import sqlmodel +sys.modules[__name__] = import_module("inkcre_core.schemas.agent") -from app.schemas.ai import AIModelID, ToolChoice - - -AgentID: typing.TypeAlias = int - -_TOOL_CHOICE_ADAPTER = pydantic.TypeAdapter(ToolChoice | None) - - -def normalize_agent_tools(value: typing.Iterable[str]) -> tuple[str, ...]: - """Validate and canonicalize Agent Tool IDs with set semantics.""" - tools = tuple(value) - if any(not tool for tool in tools): - raise ValueError("Agent Tool IDs must not be empty") - if len(tools) != len(set(tools)): - raise ValueError("Agent Tool IDs must be unique") - return tuple(sorted(tools)) - - -class AgentToolsType(sqlalchemy.TypeDecorator): - """Round-trip canonical Agent Tool ID sets through PostgreSQL text arrays.""" - - impl = sqlalchemy.dialects.postgresql.ARRAY(sqlalchemy.Text) - cache_ok = True - - def process_bind_param(self, value, dialect): - del dialect - return list(normalize_agent_tools(value or ())) - - def process_result_value(self, value, dialect): - del dialect - return tuple(value or ()) - - -class ToolChoiceType(sqlalchemy.TypeDecorator): - """Round-trip the canonical nullable ToolChoice union through JSONB.""" - - impl = sqlalchemy.dialects.postgresql.JSONB - cache_ok = True - - def load_dialect_impl(self, dialect): - return dialect.type_descriptor(sqlalchemy.dialects.postgresql.JSONB(none_as_null=True)) - - def process_bind_param(self, value, dialect): - del dialect - choice = _TOOL_CHOICE_ADAPTER.validate_python(value) - if isinstance(choice, pydantic.BaseModel): - return choice.model_dump(mode="json") - return choice - - def process_result_value(self, value, dialect): - del dialect - return _TOOL_CHOICE_ADAPTER.validate_python(value) - - -class AgentForm(pydantic.BaseModel): - """Editable Agent definition, without execution or database-managed fields.""" - - model_config = pydantic.ConfigDict(extra="forbid") - - name: str - system_prompt: str - model: AIModelID - tools: tuple[str, ...] = () - tool_choice: ToolChoice | None = None - max_model_calls_per_turn: int = pydantic.Field(gt=0) - - @pydantic.field_validator("tools") - @classmethod - def canonical_tools(cls, value: tuple[str, ...]) -> tuple[str, ...]: - return normalize_agent_tools(value) - - -class AgentUpdateForm(pydantic.BaseModel): - """PATCH omission preserves a field; only tool_choice accepts explicit null.""" - - model_config = pydantic.ConfigDict(extra="forbid") - - name: str = pydantic.Field(default=None) # pyrefly: ignore[bad-assignment] - system_prompt: str = pydantic.Field(default=None) # pyrefly: ignore[bad-assignment] - model: AIModelID = pydantic.Field(default=None) # pyrefly: ignore[bad-assignment] - tools: tuple[str, ...] = () - tool_choice: ToolChoice | None = None - max_model_calls_per_turn: int = pydantic.Field(default=None, gt=0) # pyrefly: ignore[bad-assignment] - - -class AgentDefinitionModel(sqlmodel.SQLModel, table=True): - """One reusable system-prompt, model, Tool and turn-budget composition.""" - - __tablename__ = "agents" # type: ignore - __table_args__ = ( - sqlalchemy.CheckConstraint( - "max_model_calls_per_turn > 0", - name="ck_agents_max_model_calls_per_turn_positive", - ), - ) - - id: AgentID | None = sqlmodel.Field( - default=None, - sa_column=sqlalchemy.Column( - sqlalchemy.BigInteger, - sqlalchemy.Identity(), - primary_key=True, - ), - ) - name: str = sqlmodel.Field(sa_column=sqlalchemy.Column(sqlalchemy.Text, nullable=False)) - system_prompt: str = sqlmodel.Field( - sa_column=sqlalchemy.Column(sqlalchemy.Text, nullable=False) - ) - tools: tuple[str, ...] = sqlmodel.Field( - default=(), - sa_column=sqlalchemy.Column( - AgentToolsType(), - nullable=False, - server_default=sqlalchemy.text("'{}'::text[]"), - ), - ) - tool_choice: ToolChoice | None = sqlmodel.Field( - default=None, - sa_column=sqlalchemy.Column(ToolChoiceType(), nullable=True), - ) - model: AIModelID = sqlmodel.Field( - sa_column=sqlalchemy.Column( - sqlalchemy.BigInteger, - sqlalchemy.ForeignKey("ai_models.id", onupdate="CASCADE", ondelete="RESTRICT"), - nullable=False, - ) - ) - max_model_calls_per_turn: int = sqlmodel.Field( - sa_column=sqlalchemy.Column(sqlalchemy.Integer, nullable=False) - ) - created_at: datetime.datetime = sqlmodel.Field( - default=None, - sa_column=sqlalchemy.Column( - sqlalchemy.TIMESTAMP(timezone=True), - nullable=False, - server_default=sqlalchemy.text("CURRENT_TIMESTAMP"), - ), - ) - updated_at: datetime.datetime = sqlmodel.Field( - default=None, - sa_column=sqlalchemy.Column( - sqlalchemy.TIMESTAMP(timezone=True), - nullable=False, - server_default=sqlalchemy.text("CURRENT_TIMESTAMP"), - ), - ) - - @pydantic.field_validator("tools", mode="before") - @classmethod - def canonical_tools(cls, value: typing.Iterable[str]) -> tuple[str, ...]: - return normalize_agent_tools(value) - - @pydantic.field_validator("tool_choice", mode="before") - @classmethod - def canonical_tool_choice(cls, value: typing.Any) -> ToolChoice | None: - return _TOOL_CHOICE_ADAPTER.validate_python(value) - - @pydantic.field_validator("max_model_calls_per_turn") - @classmethod - def positive_model_call_budget(cls, value: int) -> int: - if value <= 0: - raise ValueError("max_model_calls_per_turn must be positive") - return value +if TYPE_CHECKING: + from inkcre_core.schemas.agent import * diff --git a/app/schemas/ai/__init__.py b/app/schemas/ai/__init__.py index e226e18e..b76622f7 100644 --- a/app/schemas/ai/__init__.py +++ b/app/schemas/ai/__init__.py @@ -1,80 +1,3 @@ -"""Canonical AI contracts and shared persistence models.""" +"""Legacy import; use inkcre_core for new consumers.""" -from .capability import ( - AICapability, - AICapabilityType, - AIFeature, - AIModality, - AIModelCapability, - ChatCapability, - EmbeddingCapability, - normalize_capabilities, -) -from .chat import ( - AudioContentPart, - AssistantMessage, - FunctionTool, - ImageContentPart, - JSONValue, - Message, - NamedToolChoice, - SystemMessage, - TextContentPart, - ToolCall, - ToolChoice, - ToolResult, - ToolResultMessage, - UserContentPart, - UserMessage, - VideoContentPart, - validate_message_history, -) -from .embedding import BlockEmbeddingModel, RelationEmbeddingModel -from .main import ( - AIDialectID, - AIDialectModel, - AIModelID, - AIModelModel, - AIProviderID, - AIProviderModel, - EmbeddingProfileID, - EmbeddingProfileModel, -) - -__all__ = [ - "AICapability", - "AICapabilityType", - "AIDialectID", - "AIDialectModel", - "AIFeature", - "AIModality", - "AIModelCapability", - "AIModelID", - "AIModelModel", - "AIProviderID", - "AIProviderModel", - "AudioContentPart", - "AssistantMessage", - "BlockEmbeddingModel", - "ChatCapability", - "EmbeddingCapability", - "EmbeddingProfileID", - "EmbeddingProfileModel", - "FunctionTool", - "ImageContentPart", - "JSONValue", - "Message", - "NamedToolChoice", - "RelationEmbeddingModel", - "SystemMessage", - "TextContentPart", - "ToolCall", - "ToolChoice", - "ToolResult", - "ToolResultMessage", - "UserContentPart", - "UserMessage", - "VideoContentPart", - "normalize_capabilities", - "validate_message_history", -] +from inkcre_core.schemas.ai import * diff --git a/app/schemas/ai/capability.py b/app/schemas/ai/capability.py index 4a52afcf..0f97ce38 100644 --- a/app/schemas/ai/capability.py +++ b/app/schemas/ai/capability.py @@ -1,67 +1,10 @@ -"""Provider-neutral AI model capability declarations.""" +"""Legacy import; use inkcre_core for new consumers.""" -import typing +from typing import TYPE_CHECKING +from importlib import import_module +import sys -import pydantic +sys.modules[__name__] = import_module("inkcre_core.schemas.ai.capability") - -AICapabilityType: typing.TypeAlias = typing.Literal["embedding", "chat"] -AIModality: typing.TypeAlias = str -AIFeature: typing.TypeAlias = str - - -def _canonical_string_set(value: typing.Any) -> tuple[str, ...]: - if not isinstance(value, (list, tuple, set, frozenset)): - raise ValueError("value must be an array of strings") - items = tuple(value) - if not all(isinstance(item, str) and item for item in items): - raise ValueError("values must be non-empty strings") - if len(items) != len(set(items)): - raise ValueError("duplicate values are not allowed") - return tuple(sorted(items)) - - -class AICapability(pydantic.BaseModel): - """Fields shared by every typed capability declaration.""" - - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - type: AICapabilityType - input_modalities: tuple[AIModality, ...] - output_modalities: tuple[AIModality, ...] - features: tuple[AIFeature, ...] = () - - _normalize_input_modalities = pydantic.field_validator("input_modalities", mode="before")( - _canonical_string_set - ) - _normalize_output_modalities = pydantic.field_validator( - "output_modalities", mode="before" - )(_canonical_string_set) - _normalize_features = pydantic.field_validator("features", mode="before")( - _canonical_string_set - ) - - -class EmbeddingCapability(AICapability): - type: typing.Literal["embedding"] = "embedding" - - -class ChatCapability(AICapability): - type: typing.Literal["chat"] = "chat" - - -AIModelCapability: typing.TypeAlias = typing.Annotated[ - EmbeddingCapability | ChatCapability, - pydantic.Field(discriminator="type"), -] - -CAPABILITIES_ADAPTER = pydantic.TypeAdapter(tuple[AIModelCapability, ...]) - - -def normalize_capabilities(value: typing.Any) -> tuple[AIModelCapability, ...]: - """Validate and canonically order one declaration per capability type.""" - capabilities = CAPABILITIES_ADAPTER.validate_python(value) - types = tuple(capability.type for capability in capabilities) - if len(types) != len(set(types)): - raise ValueError("duplicate AI capability types are not allowed") - return tuple(sorted(capabilities, key=lambda capability: capability.type)) +if TYPE_CHECKING: + from inkcre_core.schemas.ai.capability import * diff --git a/app/schemas/ai/chat.py b/app/schemas/ai/chat.py index 013ef9ad..48402005 100644 --- a/app/schemas/ai/chat.py +++ b/app/schemas/ai/chat.py @@ -1,214 +1,10 @@ -"""Canonical AI chat, Tool and ToolResult contracts.""" +"""Legacy import; use inkcre_core for new consumers.""" -import typing -from urllib.parse import urlsplit +from typing import TYPE_CHECKING +from importlib import import_module +import sys -import pydantic +sys.modules[__name__] = import_module("inkcre_core.schemas.ai.chat") - -JSONValue: typing.TypeAlias = pydantic.JsonValue - - -class FunctionTool(pydantic.BaseModel): - """One caller-owned function Tool exposed to an AI model.""" - - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - id: str = pydantic.Field(min_length=1) - description: str - input_schema: dict[str, JSONValue] - - -class ToolCall(pydantic.BaseModel): - """One model-authored request to invoke a caller-owned Tool.""" - - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - id: str = pydantic.Field(min_length=1) - tool: str = pydantic.Field(min_length=1) - arguments: dict[str, JSONValue] - - -class ToolResult(pydantic.BaseModel): - """One caller-side result correlated to a ToolCall.""" - - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - tool_call_id: str = pydantic.Field(min_length=1) - content: JSONValue - is_error: bool = False - - -class NamedToolChoice(pydantic.BaseModel): - """Require one exact caller-owned Tool.""" - - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - type: typing.Literal["function"] = "function" - tool: str = pydantic.Field(min_length=1) - - -ToolChoice: typing.TypeAlias = typing.Literal["none", "auto", "required"] | NamedToolChoice - - -class SystemMessage(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - type: typing.Literal["system"] = "system" - content: str - - -class TextContentPart(pydantic.BaseModel): - """One authored text item in an ordered multimodal User message.""" - - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - type: typing.Literal["text"] = "text" - text: str = pydantic.Field(min_length=1) - - -class _MediaContentPart(pydantic.BaseModel): - """AI-owned media transfer value, independent of Block and Storage identity.""" - - model_config = pydantic.ConfigDict( - extra="forbid", - frozen=True, - ser_json_bytes="base64", - val_json_bytes="base64", - ) - - data: bytes = pydantic.Field(min_length=1) - mime_type: str = pydantic.Field(min_length=1) - transfer_url: str | None = None - - @pydantic.field_validator("mime_type") - @classmethod - def normalized_mime_type(cls, value: str) -> str: - normalized = value.partition(";")[0].strip().lower() - if "/" not in normalized or any(character.isspace() for character in normalized): - raise ValueError("mime_type must be one concrete media type") - return normalized - - @pydantic.field_validator("transfer_url") - @classmethod - def http_transfer_url(cls, value: str | None) -> str | None: - if value is None: - return None - parsed = urlsplit(value) - if parsed.scheme not in {"http", "https"} or not parsed.netloc: - raise ValueError("transfer_url must be one absolute HTTP(S) URL") - if parsed.username is not None or parsed.password is not None: - raise ValueError("transfer_url must not contain user-info credentials") - return value - - -class ImageContentPart(_MediaContentPart): - type: typing.Literal["image"] = "image" - - @pydantic.field_validator("mime_type") - @classmethod - def image_mime_type(cls, value: str) -> str: - if not value.startswith("image/"): - raise ValueError("ImageContentPart requires an image MIME type") - return value - - -class AudioContentPart(_MediaContentPart): - type: typing.Literal["audio"] = "audio" - - @pydantic.field_validator("mime_type") - @classmethod - def audio_mime_type(cls, value: str) -> str: - if not value.startswith("audio/"): - raise ValueError("AudioContentPart requires an audio MIME type") - return value - - -class VideoContentPart(_MediaContentPart): - type: typing.Literal["video"] = "video" - - @pydantic.field_validator("mime_type") - @classmethod - def video_mime_type(cls, value: str) -> str: - if not value.startswith("video/"): - raise ValueError("VideoContentPart requires a video MIME type") - return value - - -UserContentPart: typing.TypeAlias = typing.Annotated[ - TextContentPart | ImageContentPart | AudioContentPart | VideoContentPart, - pydantic.Field(discriminator="type"), -] - - -class UserMessage(pydantic.BaseModel): - model_config = pydantic.ConfigDict( - extra="forbid", - frozen=True, - ser_json_bytes="base64", - val_json_bytes="base64", - ) - - type: typing.Literal["user"] = "user" - content: tuple[UserContentPart, ...] = pydantic.Field(min_length=1) - - -class AssistantMessage(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - type: typing.Literal["assistant"] = "assistant" - content: str | None = None - tool_calls: tuple[ToolCall, ...] = () - - @pydantic.model_validator(mode="after") - def unique_tool_call_ids(self) -> "AssistantMessage": - call_ids = tuple(call.id for call in self.tool_calls) - if len(call_ids) != len(set(call_ids)): - raise ValueError("AssistantMessage ToolCall IDs must be unique") - return self - - -class ToolResultMessage(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - type: typing.Literal["tool_result"] = "tool_result" - results: tuple[ToolResult, ...] = pydantic.Field(min_length=1) - - @pydantic.model_validator(mode="after") - def unique_tool_call_ids(self) -> "ToolResultMessage": - call_ids = tuple(result.tool_call_id for result in self.results) - if len(call_ids) != len(set(call_ids)): - raise ValueError("ToolResultMessage ToolCall IDs must be unique") - return self - - -Message: typing.TypeAlias = typing.Annotated[ - SystemMessage | UserMessage | AssistantMessage | ToolResultMessage, - pydantic.Field(discriminator="type"), -] - -_MESSAGES_ADAPTER = pydantic.TypeAdapter(tuple[Message, ...]) - - -def validate_message_history(value: typing.Any) -> tuple[Message, ...]: - """Validate canonical types and every closed Assistant/ToolResult pair.""" - messages = _MESSAGES_ADAPTER.validate_python(value) - index = 0 - while index < len(messages): - message = messages[index] - if isinstance(message, ToolResultMessage): - raise ValueError("ToolResultMessage must follow an AssistantMessage with ToolCalls") - if isinstance(message, AssistantMessage) and message.tool_calls: - if index + 1 >= len(messages) or not isinstance( - messages[index + 1], ToolResultMessage - ): - raise ValueError("AssistantMessage ToolCalls require an adjacent ToolResultMessage") - result_message = typing.cast(ToolResultMessage, messages[index + 1]) - call_ids = {call.id for call in message.tool_calls} - result_ids = {result.tool_call_id for result in result_message.results} - if result_ids != call_ids: - raise ValueError("ToolResultMessage must exactly cover preceding ToolCall IDs") - index += 2 - continue - index += 1 - return messages +if TYPE_CHECKING: + from inkcre_core.schemas.ai.chat import * diff --git a/app/schemas/ai/embedding.py b/app/schemas/ai/embedding.py index 061d6d28..7ca31a95 100644 --- a/app/schemas/ai/embedding.py +++ b/app/schemas/ai/embedding.py @@ -1,91 +1,10 @@ -"""Profile-scoped Block and Relation embedding records.""" +"""Legacy import; use inkcre_core for new consumers.""" -import datetime +from typing import TYPE_CHECKING +from importlib import import_module +import sys -import pgvector.sqlalchemy -import sqlalchemy -import sqlmodel +sys.modules[__name__] = import_module("inkcre_core.schemas.ai.embedding") -from app.schemas.info_base.main import Vector - -from .main import EmbeddingProfileID - - -class BlockEmbeddingModel(sqlmodel.SQLModel, table=True): - __tablename__ = "block_embeddings" # type: ignore - - profile: EmbeddingProfileID = sqlmodel.Field( - sa_column=sqlalchemy.Column( - sqlalchemy.BigInteger, - sqlalchemy.ForeignKey( - "embedding_profiles.id", onupdate="CASCADE", ondelete="RESTRICT" - ), - primary_key=True, - ) - ) - block: int = sqlmodel.Field( - sa_column=sqlalchemy.Column( - sqlalchemy.Integer, - sqlalchemy.ForeignKey("blocks.id", onupdate="CASCADE", ondelete="CASCADE"), - primary_key=True, - ) - ) - embedding: Vector = sqlmodel.Field( - sa_column=sqlalchemy.Column(pgvector.sqlalchemy.VECTOR(), nullable=False) - ) - created_at: datetime.datetime = sqlmodel.Field( - default=None, - sa_column=sqlalchemy.Column( - sqlalchemy.TIMESTAMP(timezone=True), - nullable=False, - server_default=sqlalchemy.text("CURRENT_TIMESTAMP"), - ), - ) - updated_at: datetime.datetime = sqlmodel.Field( - default=None, - sa_column=sqlalchemy.Column( - sqlalchemy.TIMESTAMP(timezone=True), - nullable=False, - server_default=sqlalchemy.text("CURRENT_TIMESTAMP"), - ), - ) - - -class RelationEmbeddingModel(sqlmodel.SQLModel, table=True): - __tablename__ = "relation_embeddings" # type: ignore - - profile: EmbeddingProfileID = sqlmodel.Field( - sa_column=sqlalchemy.Column( - sqlalchemy.BigInteger, - sqlalchemy.ForeignKey( - "embedding_profiles.id", onupdate="CASCADE", ondelete="RESTRICT" - ), - primary_key=True, - ) - ) - relation: int = sqlmodel.Field( - sa_column=sqlalchemy.Column( - sqlalchemy.Integer, - sqlalchemy.ForeignKey("relations.id", onupdate="CASCADE", ondelete="CASCADE"), - primary_key=True, - ) - ) - embedding: Vector = sqlmodel.Field( - sa_column=sqlalchemy.Column(pgvector.sqlalchemy.VECTOR(), nullable=False) - ) - created_at: datetime.datetime = sqlmodel.Field( - default=None, - sa_column=sqlalchemy.Column( - sqlalchemy.TIMESTAMP(timezone=True), - nullable=False, - server_default=sqlalchemy.text("CURRENT_TIMESTAMP"), - ), - ) - updated_at: datetime.datetime = sqlmodel.Field( - default=None, - sa_column=sqlalchemy.Column( - sqlalchemy.TIMESTAMP(timezone=True), - nullable=False, - server_default=sqlalchemy.text("CURRENT_TIMESTAMP"), - ), - ) +if TYPE_CHECKING: + from inkcre_core.schemas.ai.embedding import * diff --git a/app/schemas/ai/main.py b/app/schemas/ai/main.py index f189d7eb..62ee2ba7 100644 --- a/app/schemas/ai/main.py +++ b/app/schemas/ai/main.py @@ -1,227 +1,10 @@ -"""Shared AI dialect, provider, model and embedding-profile facts.""" +"""Legacy import; use inkcre_core for new consumers.""" -import datetime -import typing +from typing import TYPE_CHECKING +from importlib import import_module +import sys -import pydantic -import sqlalchemy -import sqlalchemy.dialects.postgresql -import sqlmodel +sys.modules[__name__] = import_module("inkcre_core.schemas.ai.main") -from .capability import CAPABILITIES_ADAPTER, AIModelCapability, normalize_capabilities - - -AIDialectID: typing.TypeAlias = str -AIProviderID: typing.TypeAlias = int -AIModelID: typing.TypeAlias = int -EmbeddingProfileID: typing.TypeAlias = int - - -class AICapabilitiesType(sqlalchemy.TypeDecorator): - """Round-trip typed, canonical capability declarations through JSONB.""" - - impl = sqlalchemy.dialects.postgresql.JSONB - cache_ok = True - - def process_bind_param(self, value, dialect): - del dialect - return [ - capability.model_dump(mode="json") for capability in normalize_capabilities(value) - ] - - def process_result_value(self, value, dialect): - del dialect - return CAPABILITIES_ADAPTER.validate_python(value or ()) - - -class AIDialectModel(sqlmodel.SQLModel, table=True): - """One exact shared AI wire/config dialect contract.""" - - __tablename__ = "ai_dialects" # type: ignore - - id: AIDialectID = sqlmodel.Field( - sa_column=sqlalchemy.Column(sqlalchemy.Text, primary_key=True) - ) - description: str = sqlmodel.Field( - sa_column=sqlalchemy.Column(sqlalchemy.Text, nullable=False) - ) - config_schema: dict[str, typing.Any] = sqlmodel.Field( - default_factory=dict, - sa_column=sqlalchemy.Column( - sqlalchemy.dialects.postgresql.JSONB, - nullable=False, - server_default=sqlalchemy.text("'{}'::jsonb"), - ), - ) - - -class AIProviderModel(sqlmodel.SQLModel, table=True): - """One deployment-scoped configuration of an AI dialect.""" - - __tablename__ = "ai_providers" # type: ignore - - id: AIProviderID | None = sqlmodel.Field( - default=None, - sa_column=sqlalchemy.Column( - sqlalchemy.BigInteger, - sqlalchemy.Identity(), - primary_key=True, - ), - ) - name: str = sqlmodel.Field(sa_column=sqlalchemy.Column(sqlalchemy.Text, nullable=False)) - dialect: AIDialectID = sqlmodel.Field( - sa_column=sqlalchemy.Column( - sqlalchemy.Text, - sqlalchemy.ForeignKey("ai_dialects.id", onupdate="CASCADE", ondelete="RESTRICT"), - nullable=False, - ) - ) - config: dict[str, typing.Any] = sqlmodel.Field( - default_factory=dict, - sa_column=sqlalchemy.Column( - sqlalchemy.dialects.postgresql.JSONB, - nullable=False, - server_default=sqlalchemy.text("'{}'::jsonb"), - ), - ) - enabled: bool = sqlmodel.Field( - default=True, - sa_column=sqlalchemy.Column( - sqlalchemy.Boolean, - nullable=False, - server_default=sqlalchemy.true(), - ), - ) - created_at: datetime.datetime = sqlmodel.Field( - default=None, - sa_column=sqlalchemy.Column( - sqlalchemy.TIMESTAMP(timezone=True), - nullable=False, - server_default=sqlalchemy.text("CURRENT_TIMESTAMP"), - ), - ) - updated_at: datetime.datetime = sqlmodel.Field( - default=None, - sa_column=sqlalchemy.Column( - sqlalchemy.TIMESTAMP(timezone=True), - nullable=False, - server_default=sqlalchemy.text("CURRENT_TIMESTAMP"), - ), - ) - - -class AIModelModel(sqlmodel.SQLModel, table=True): - """One provider-bound native model offering and its effective capabilities.""" - - __tablename__ = "ai_models" # type: ignore - __table_args__ = ( - sqlalchemy.UniqueConstraint( - "provider", "native_model_id", name="uq_ai_models_provider_native_model" - ), - ) - - id: AIModelID | None = sqlmodel.Field( - default=None, - sa_column=sqlalchemy.Column( - sqlalchemy.BigInteger, - sqlalchemy.Identity(), - primary_key=True, - ), - ) - provider: AIProviderID = sqlmodel.Field( - sa_column=sqlalchemy.Column( - sqlalchemy.BigInteger, - sqlalchemy.ForeignKey("ai_providers.id", onupdate="CASCADE", ondelete="RESTRICT"), - nullable=False, - ) - ) - native_model_id: str = sqlmodel.Field( - sa_column=sqlalchemy.Column(sqlalchemy.Text, nullable=False) - ) - name: str | None = sqlmodel.Field( - default=None, - sa_column=sqlalchemy.Column(sqlalchemy.Text, nullable=True), - ) - capabilities: tuple[AIModelCapability, ...] = sqlmodel.Field( - default=(), - sa_column=sqlalchemy.Column(AICapabilitiesType(), nullable=False), - ) - enabled: bool = sqlmodel.Field( - default=True, - sa_column=sqlalchemy.Column( - sqlalchemy.Boolean, - nullable=False, - server_default=sqlalchemy.true(), - ), - ) - created_at: datetime.datetime = sqlmodel.Field( - default=None, - sa_column=sqlalchemy.Column( - sqlalchemy.TIMESTAMP(timezone=True), - nullable=False, - server_default=sqlalchemy.text("CURRENT_TIMESTAMP"), - ), - ) - updated_at: datetime.datetime = sqlmodel.Field( - default=None, - sa_column=sqlalchemy.Column( - sqlalchemy.TIMESTAMP(timezone=True), - nullable=False, - server_default=sqlalchemy.text("CURRENT_TIMESTAMP"), - ), - ) - - @pydantic.field_validator("capabilities", mode="before") - @classmethod - def canonical_capabilities(cls, value: typing.Any) -> tuple[AIModelCapability, ...]: - return normalize_capabilities(value) - - -class EmbeddingProfileModel(sqlmodel.SQLModel, table=True): - """One durable vector-space compatibility contract.""" - - __tablename__ = "embedding_profiles" # type: ignore - __table_args__ = ( - sqlalchemy.CheckConstraint( - "dimensions > 0", name="ck_embedding_profiles_dimensions_positive" - ), - ) - - id: EmbeddingProfileID | None = sqlmodel.Field( - default=None, - sa_column=sqlalchemy.Column( - sqlalchemy.BigInteger, - sqlalchemy.Identity(), - primary_key=True, - ), - ) - name: str | None = sqlmodel.Field( - default=None, - sa_column=sqlalchemy.Column(sqlalchemy.Text, nullable=True), - ) - ai_model: AIModelID = sqlmodel.Field( - sa_column=sqlalchemy.Column( - sqlalchemy.BigInteger, - sqlalchemy.ForeignKey("ai_models.id", onupdate="CASCADE", ondelete="RESTRICT"), - nullable=False, - ) - ) - dimensions: int = sqlmodel.Field( - sa_column=sqlalchemy.Column(sqlalchemy.Integer, nullable=False) - ) - created_at: datetime.datetime = sqlmodel.Field( - default=None, - sa_column=sqlalchemy.Column( - sqlalchemy.TIMESTAMP(timezone=True), - nullable=False, - server_default=sqlalchemy.text("CURRENT_TIMESTAMP"), - ), - ) - updated_at: datetime.datetime = sqlmodel.Field( - default=None, - sa_column=sqlalchemy.Column( - sqlalchemy.TIMESTAMP(timezone=True), - nullable=False, - server_default=sqlalchemy.text("CURRENT_TIMESTAMP"), - ), - ) +if TYPE_CHECKING: + from inkcre_core.schemas.ai.main import * diff --git a/app/schemas/cron.py b/app/schemas/cron.py index bf452e15..1fa18e66 100644 --- a/app/schemas/cron.py +++ b/app/schemas/cron.py @@ -1,120 +1,10 @@ -"""Deployment-wide recurring Job-template schemas.""" +"""Legacy import; use inkcre_core for new consumers.""" -import datetime -import typing +from typing import TYPE_CHECKING +from importlib import import_module +import sys -import pydantic -import sqlalchemy -import sqlalchemy.dialects.postgresql -import sqlmodel +sys.modules[__name__] = import_module("inkcre_core.schemas.cron") -from .job import JobID, JobTypeID - - -CronID: typing.TypeAlias = int - - -class CronModel(sqlmodel.SQLModel, table=True): - """One five-field UNIX schedule that materializes typed Jobs.""" - - __tablename__ = "crons" # type: ignore - __table_args__ = ( - sqlalchemy.Index("crons_enabled_idx", "enabled"), - sqlalchemy.CheckConstraint( - "job_timeout_seconds IS NULL OR job_timeout_seconds > 0", - name="crons_job_timeout_seconds_positive", - ), - ) - - id: CronID | None = sqlmodel.Field( - default=None, - sa_column=sqlalchemy.Column( - sqlalchemy.BigInteger, - sqlalchemy.Identity(), - primary_key=True, - ), - ) - schedule: str = sqlmodel.Field( - min_length=1, - sa_column=sqlalchemy.Column(sqlalchemy.Text, nullable=False), - ) - enabled: bool = sqlmodel.Field( - default=True, - sa_column=sqlalchemy.Column( - sqlalchemy.Boolean, - nullable=False, - server_default=sqlalchemy.true(), - ), - ) - job_type: JobTypeID = sqlmodel.Field( - sa_column=sqlalchemy.Column( - sqlalchemy.Text, - sqlalchemy.ForeignKey("job_types.id", onupdate="CASCADE", ondelete="RESTRICT"), - nullable=False, - ) - ) - job_parameters: dict[str, typing.Any] = sqlmodel.Field( - default_factory=dict, - sa_column=sqlalchemy.Column( - sqlalchemy.dialects.postgresql.JSONB, - nullable=False, - server_default=sqlalchemy.text("'{}'::jsonb"), - ), - ) - job_timeout_seconds: int | None = sqlmodel.Field( - default=None, - gt=0, - sa_column=sqlalchemy.Column(sqlalchemy.Integer, nullable=True), - ) - last_job: JobID | None = sqlmodel.Field( - default=None, - sa_column=sqlalchemy.Column( - sqlalchemy.BigInteger, - sqlalchemy.ForeignKey("jobs.id", onupdate="CASCADE", ondelete="RESTRICT"), - nullable=True, - ), - ) - last_scheduled_for: datetime.datetime | None = sqlmodel.Field( - default=None, - sa_column=sqlalchemy.Column(sqlalchemy.TIMESTAMP(timezone=True), nullable=True), - ) - created_at: datetime.datetime = sqlmodel.Field( - default=None, - sa_column=sqlalchemy.Column( - sqlalchemy.TIMESTAMP(timezone=True), - nullable=False, - server_default=sqlalchemy.text("CURRENT_TIMESTAMP"), - ), - ) - updated_at: datetime.datetime = sqlmodel.Field( - default=None, - sa_column=sqlalchemy.Column( - sqlalchemy.TIMESTAMP(timezone=True), - nullable=False, - server_default=sqlalchemy.text("CURRENT_TIMESTAMP"), - ), - ) - - -class CronForm(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid") - - schedule: str = pydantic.Field(min_length=1) - enabled: bool = True - job_type: JobTypeID - job_parameters: dict[str, typing.Any] = pydantic.Field(default_factory=dict) - job_timeout_seconds: int | None = pydantic.Field(default=None, gt=0) - - -class CronUpdateForm(pydantic.BaseModel): - """Optional submitted fields; null only clears the timeout override.""" - - model_config = pydantic.ConfigDict(extra="forbid") - - # Omission is not null input: exclude_unset removes these defaults, whereas - # an explicitly submitted null still fails the non-null field type. - schedule: str = pydantic.Field(default=None, min_length=1) # pyrefly: ignore[bad-assignment] - enabled: bool = pydantic.Field(default=None) # pyrefly: ignore[bad-assignment] - job_type: JobTypeID = pydantic.Field(default=None) # pyrefly: ignore[bad-assignment] - job_parameters: dict[str, typing.Any] = pydantic.Field(default_factory=dict) - job_timeout_seconds: int | None = pydantic.Field(default=None, gt=0) +if TYPE_CHECKING: + from inkcre_core.schemas.cron import * diff --git a/app/schemas/deployment_config.py b/app/schemas/deployment_config.py index 0c4e4c61..f11766ff 100644 --- a/app/schemas/deployment_config.py +++ b/app/schemas/deployment_config.py @@ -1,69 +1,10 @@ -"""Deployment-scoped shared configuration persistence and API models.""" +"""Legacy import; use inkcre_core for new consumers.""" -import datetime -import typing +from typing import TYPE_CHECKING +from importlib import import_module +import sys -import pydantic -import sqlalchemy -import sqlalchemy.dialects.postgresql -import sqlmodel +sys.modules[__name__] = import_module("inkcre_core.schemas.deployment_config") - -DeploymentConfigKey: typing.TypeAlias = str -DeploymentConfigSchemaID: typing.TypeAlias = str - - -class DeploymentConfigModel(sqlmodel.SQLModel, table=True): - """One deployment-owned config value under an exact schema contract.""" - - __tablename__ = "configs" # type: ignore - - key: DeploymentConfigKey = sqlmodel.Field( - sa_column=sqlalchemy.Column(sqlalchemy.Text, primary_key=True) - ) - schema_id: DeploymentConfigSchemaID = sqlmodel.Field( - sa_column=sqlalchemy.Column("schema", sqlalchemy.Text, nullable=False) - ) - value: dict[str, typing.Any] = sqlmodel.Field( - sa_column=sqlalchemy.Column( - sqlalchemy.dialects.postgresql.JSONB, - nullable=False, - ) - ) - created_at: datetime.datetime = sqlmodel.Field( - default=None, - sa_column=sqlalchemy.Column( - sqlalchemy.TIMESTAMP(timezone=True), - nullable=False, - server_default=sqlalchemy.text("CURRENT_TIMESTAMP"), - ), - ) - updated_at: datetime.datetime = sqlmodel.Field( - default=None, - sa_column=sqlalchemy.Column( - sqlalchemy.TIMESTAMP(timezone=True), - nullable=False, - server_default=sqlalchemy.text("CURRENT_TIMESTAMP"), - ), - ) - - -class DeploymentConfigReplaceForm(pydantic.BaseModel): - """Complete PUT body; schema changes are allowed only through this form.""" - - model_config = pydantic.ConfigDict(populate_by_name=True) - - schema_id: DeploymentConfigSchemaID = pydantic.Field(alias="schema") - value: dict[str, typing.Any] - - -class DeploymentConfigView(pydantic.BaseModel): - """Stored deployment config and metadata; reading does not revalidate value.""" - - model_config = pydantic.ConfigDict(populate_by_name=True) - - key: DeploymentConfigKey - schema_id: DeploymentConfigSchemaID = pydantic.Field(alias="schema") - value: dict[str, typing.Any] - created_at: datetime.datetime - updated_at: datetime.datetime +if TYPE_CHECKING: + from inkcre_core.schemas.deployment_config import * diff --git a/app/schemas/extension/__init__.py b/app/schemas/extension/__init__.py index 4a6fac60..de2f6925 100644 --- a/app/schemas/extension/__init__.py +++ b/app/schemas/extension/__init__.py @@ -1,37 +1,3 @@ -from .main import ( - DisableExtensionCommand, - EnableExtensionCommand, - ExtensionPeerStatus, - ExtensionPeerStatusRow, - EXTENSION_NAME_PATTERN, - EXTENSION_SEMVER_PATTERN, - ExtensionManagementCommand, - ExtensionModel, - ExtensionPeerParticipationModel, - ExtensionName, - InstallExtensionCommand, - PatchExtensionConfigCommand, - SelectExtensionInput, - SelectExtensionCommand, - ExtensionStatusCommand, - UnselectExtensionCommand, -) +"""Legacy import; use inkcre_core for new consumers.""" -__all__ = [ - "DisableExtensionCommand", - "EnableExtensionCommand", - "ExtensionPeerStatus", - "ExtensionPeerStatusRow", - "EXTENSION_NAME_PATTERN", - "EXTENSION_SEMVER_PATTERN", - "ExtensionManagementCommand", - "ExtensionModel", - "ExtensionPeerParticipationModel", - "ExtensionName", - "InstallExtensionCommand", - "PatchExtensionConfigCommand", - "SelectExtensionInput", - "SelectExtensionCommand", - "ExtensionStatusCommand", - "UnselectExtensionCommand", -] +from inkcre_core.schemas.extension import * diff --git a/app/schemas/extension/main.py b/app/schemas/extension/main.py index 0d6322ef..cc410f7c 100644 --- a/app/schemas/extension/main.py +++ b/app/schemas/extension/main.py @@ -1,215 +1,10 @@ -import typing -import uuid +"""Legacy import; use inkcre_core for new consumers.""" -import pydantic -import sqlalchemy -import sqlalchemy.dialects.postgresql -import sqlmodel +from typing import TYPE_CHECKING +from importlib import import_module +import sys +sys.modules[__name__] = import_module("inkcre_core.schemas.extension.main") -ExtensionName: typing.TypeAlias = str -NativeDistribution = typing.Annotated[str, pydantic.Field(min_length=1)] -EXTENSION_NAME_PATTERN = ( - r"^[a-z0-9]([a-z0-9-]{0,62}[a-z0-9])?/" - r"[a-z0-9]([a-z0-9-]{0,62}[a-z0-9])?$" -) -EXTENSION_SEMVER_PATTERN = ( - r"^(0|[1-9][0-9]*)[.](0|[1-9][0-9]*)[.](0|[1-9][0-9]*)" - r"(-(0|[1-9][0-9]*|[0-9]*[A-Za-z-][0-9A-Za-z-]*)" - r"([.](0|[1-9][0-9]*|[0-9]*[A-Za-z-][0-9A-Za-z-]*))*)?$" -) - - -class InstallExtensionCommand(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - action: typing.Literal["install"] - extension: ExtensionName - version: str = pydantic.Field(pattern=EXTENSION_SEMVER_PATTERN) - - -class SelectExtensionCommand(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - action: typing.Literal["select"] - extension: ExtensionName - distribution: NativeDistribution - version: str = pydantic.Field(pattern=EXTENSION_SEMVER_PATTERN) - - -class ExtensionStatusCommand(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - action: typing.Literal["status"] - extension: ExtensionName | None = None - - -class UnselectExtensionCommand(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - action: typing.Literal["unselect"] - extension: ExtensionName - - -class EnableExtensionCommand(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - action: typing.Literal["enable"] - extension: ExtensionName - - -class DisableExtensionCommand(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - action: typing.Literal["disable"] - extension: ExtensionName - - -class PatchExtensionConfigCommand(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - action: typing.Literal["patch_config"] - extension: ExtensionName - patch: dict[str, typing.Any] - - -class ExtensionPeerStatusRow(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - name: ExtensionName - peer_id: uuid.UUID - distribution: NativeDistribution | None - version: str | None - enabled: bool - config_schema: dict[str, typing.Any] | None - supported_distributions: tuple[NativeDistribution, ...] - loaded_version: str | None - running: bool - restart_required: bool - operation: str | None - error: str | None - - -class ExtensionPeerStatus(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - peer_id: uuid.UUID - supported_distributions: tuple[NativeDistribution, ...] - extensions: tuple[ExtensionPeerStatusRow, ...] - - -class SelectExtensionInput(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - distribution: NativeDistribution - version: str = pydantic.Field(pattern=EXTENSION_SEMVER_PATTERN) - - -ExtensionManagementCommand: typing.TypeAlias = typing.Annotated[ - SelectExtensionCommand - | ExtensionStatusCommand - | UnselectExtensionCommand - | EnableExtensionCommand - | DisableExtensionCommand - | PatchExtensionConfigCommand, - pydantic.Field(discriminator="action"), -] - - -class ExtensionModel(sqlmodel.SQLModel, table=True): - """Shared Extension identity, configuration, and producer-owned state.""" - - __tablename__: str = "extensions" # type: ignore - __table_args__ = ( - sqlalchemy.CheckConstraint( - f"name ~ '{EXTENSION_NAME_PATTERN}'", - name="extensions_name_canonical", - ), - sqlalchemy.CheckConstraint( - "jsonb_typeof(state) = 'object'", - name="extensions_state_object", - ), - ) - - name: ExtensionName = sqlmodel.Field( - sa_column=sqlalchemy.Column(sqlalchemy.Text, primary_key=True) - ) - nickname: str | None = sqlmodel.Field( - default=None, - sa_column=sqlalchemy.Column(sqlalchemy.Text, nullable=True), - ) - config: dict = sqlmodel.Field( - default_factory=dict, - sa_column=sqlalchemy.Column( - sqlalchemy.dialects.postgresql.JSONB, - server_default=sqlalchemy.text("'{}'::jsonb"), - nullable=False, - ), - ) - state: dict = sqlmodel.Field( - default_factory=dict, - sa_column=sqlalchemy.Column( - sqlalchemy.dialects.postgresql.JSONB, - server_default=sqlalchemy.text("'{}'::jsonb"), - nullable=False, - ), - ) - - -class ExtensionPeerParticipationModel(sqlmodel.SQLModel, table=True): - """One Peer’s exact native distribution selection and enable intent.""" - - __tablename__: str = "extension_peer_participations" # type: ignore - __table_args__ = ( - sqlalchemy.CheckConstraint( - f"name ~ '{EXTENSION_NAME_PATTERN}'", - name="extension_peer_participations_name_canonical", - ), - sqlalchemy.CheckConstraint( - f"version ~ '{EXTENSION_SEMVER_PATTERN}'", - name="extension_peer_participations_version_canonical", - ), - sqlalchemy.CheckConstraint( - "config_schema IS NULL OR jsonb_typeof(config_schema) = 'object'", - name="extension_peer_participations_config_schema_object", - ), - sqlalchemy.CheckConstraint( - "distribution IS NOT NULL OR config_schema IS NULL", - name="extension_peer_participations_unresolved_schema_null", - ), - sqlalchemy.ForeignKeyConstraint( - ["name"], ["inkcre.extensions.name"], ondelete="CASCADE" - ), - sqlalchemy.ForeignKeyConstraint(["peer_id"], ["inkcre.peers.id"], ondelete="CASCADE"), - sqlalchemy.PrimaryKeyConstraint( - "name", "peer_id", name="extension_peer_participations_pkey" - ), - ) - - name: ExtensionName = sqlmodel.Field( - sa_column=sqlalchemy.Column(sqlalchemy.Text, nullable=False) - ) - peer_id: uuid.UUID = sqlmodel.Field( - sa_column=sqlalchemy.Column( - sqlalchemy.dialects.postgresql.UUID(as_uuid=True), nullable=False - ) - ) - distribution: str | None = sqlmodel.Field( - default=None, - sa_column=sqlalchemy.Column(sqlalchemy.Text, nullable=True), - ) - version: str = sqlmodel.Field( - sa_column=sqlalchemy.Column(sqlalchemy.Text, nullable=False) - ) - enabled: bool = sqlmodel.Field( - default=False, - sa_column=sqlalchemy.Column( - sqlalchemy.Boolean, nullable=False, server_default=sqlalchemy.false() - ), - ) - config_schema: dict | None = sqlmodel.Field( - default=None, - sa_column=sqlalchemy.Column( - sqlalchemy.dialects.postgresql.JSONB(none_as_null=True), nullable=True - ), - ) +if TYPE_CHECKING: + from inkcre_core.schemas.extension.main import * diff --git a/app/schemas/graph_navigation_retrieval.py b/app/schemas/graph_navigation_retrieval.py index c141619e..4dd32978 100644 --- a/app/schemas/graph_navigation_retrieval.py +++ b/app/schemas/graph_navigation_retrieval.py @@ -1,90 +1,10 @@ -"""Presentation-neutral graph-navigation retrieval contracts.""" +"""Legacy import; use inkcre_core for new consumers.""" -import typing +from typing import TYPE_CHECKING +from importlib import import_module +import sys -import pydantic +sys.modules[__name__] = import_module("inkcre_core.schemas.graph_navigation_retrieval") -from app.schemas.info_base.block import BlockID, BlockModel -from app.schemas.info_base.relation import RelationID, RelationModel - - -GraphDirection: typing.TypeAlias = typing.Literal["in", "out", "both"] - -DEFAULT_NEIGHBORHOOD_LIMIT = 20 -MAX_NEIGHBORHOOD_LIMIT = 100 -DEFAULT_MAX_HOPS = 4 -MAX_MAX_HOPS = 8 -DEFAULT_MAX_EXPLORED_BLOCKS = 1000 -MAX_MAX_EXPLORED_BLOCKS = 10000 -DEFAULT_MAX_EXPLORED_RELATIONS = 10000 - - -class GraphModel(pydantic.BaseModel): - """Endpoint-closed persisted graph read model.""" - - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - blocks: tuple[BlockModel, ...] - relations: tuple[RelationModel, ...] - - -class BlockNeighborhood(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - focal_block: BlockID - graph: GraphModel - next_cursor: RelationID | None = None - - -class RelationNeighborhood(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - focal_relation: RelationID - graph: GraphModel - - -class ConnectedSeedComponent(pydantic.BaseModel): - """One observed undirected component containing one or more input seeds.""" - - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - seed_block_ids: tuple[BlockID, ...] - member_block_ids: tuple[BlockID, ...] - - -class ConnectedComponentsResult(pydantic.BaseModel): - """Bounded seed partition plus an endpoint-closed spanning proof.""" - - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - components: tuple[ConnectedSeedComponent, ...] - proof_graph: GraphModel - missing_seed_block_ids: tuple[BlockID, ...] - truncated: bool - - -class PathFound(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - status: typing.Literal["found"] = "found" - graph: GraphModel - block_path: tuple[BlockID, ...] - relation_path: tuple[RelationID, ...] - - -class PathNotFound(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - status: typing.Literal["not_found"] = "not_found" - - -class PathLimitReached(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - status: typing.Literal["limit_reached"] = "limit_reached" - - -PathResult: typing.TypeAlias = typing.Annotated[ - PathFound | PathNotFound | PathLimitReached, - pydantic.Field(discriminator="status"), -] +if TYPE_CHECKING: + from inkcre_core.schemas.graph_navigation_retrieval import * diff --git a/app/schemas/info_base/__init__.py b/app/schemas/info_base/__init__.py index abcd96c5..c596294e 100644 --- a/app/schemas/info_base/__init__.py +++ b/app/schemas/info_base/__init__.py @@ -1,36 +1,3 @@ -from .block import BlockForm, BlockModel, BlockID, ResolverType -from .relation import RelationCreateForm, RelationForm, RelationModel, RelationID -from .storage import StorageBlobModel, StorageModel, StorageID, StorageTypesModel -from .main import ( - GraphBlockForm, - GraphForm, - GraphRelationForm, - InArcForm, - OutArcForm, - StarsGraphForm, - SubmitGraphResult, - Vector, -) +"""Legacy import; use inkcre_core for new consumers.""" -__all__ = [ - "BlockModel", - "BlockForm", - "BlockID", - "ResolverType", - "RelationModel", - "RelationForm", - "RelationCreateForm", - "RelationID", - "StorageModel", - "StorageBlobModel", - "StorageID", - "StorageTypesModel", - "StarsGraphForm", - "OutArcForm", - "InArcForm", - "GraphBlockForm", - "GraphRelationForm", - "GraphForm", - "SubmitGraphResult", - "Vector", -] +from inkcre_core.schemas.info_base import * diff --git a/app/schemas/info_base/block.py b/app/schemas/info_base/block.py index ffa3f8b4..b794ecd3 100644 --- a/app/schemas/info_base/block.py +++ b/app/schemas/info_base/block.py @@ -1,111 +1,10 @@ -import datetime -import typing -import sqlalchemy -import sqlmodel -from pydantic import PrivateAttr -from typing import Optional as Opt -from .storage import StorageID +"""Legacy import; use inkcre_core for new consumers.""" -ResolverType: typing.TypeAlias = str -"""Type of resolver. +from typing import TYPE_CHECKING +from importlib import import_module +import sys -Extension resolvers should follows `extensions.{extension_id}.{resolver_name}` pattern. -""" -BlockID: typing.TypeAlias = int -HydratedContent: typing.TypeAlias = str | bytes -ResolverTV = typing.TypeVar("ResolverTV", bound="ResolverType") -"""Resolver type variable""" +sys.modules[__name__] = import_module("inkcre_core.schemas.info_base.block") - -class BlockForm(sqlmodel.SQLModel): - """Producer-owned values for creating one Block.""" - - model_config = {"extra": "forbid"} - - storage: Opt[StorageID] = sqlmodel.Field( - default=None, - sa_column=sqlalchemy.Column( - sqlalchemy.Integer, - sqlalchemy.ForeignKey("storages.id", onupdate="CASCADE", ondelete="RESTRICT"), - nullable=True, - ), - ) - resolver: ResolverType = sqlmodel.Field( - sa_column=sqlalchemy.Column(sqlalchemy.Text, nullable=False) - ) - content: str = sqlmodel.Field( - sa_column=sqlalchemy.Column(sqlalchemy.Text, nullable=False) - ) - """Content of the block. - - Block content stored as text in database. - For runtime usage, use storage to get the raw content. - """ - - -class BlockModel(BlockForm, table=True): - __tablename__ = "blocks" # type: ignore - - id: Opt[BlockID] = sqlmodel.Field( - sa_column=sqlmodel.Column(sqlmodel.Integer, primary_key=True, autoincrement=True), - default=None, - ) - created_at: datetime.datetime = sqlmodel.Field( - default=None, - sa_column=sqlalchemy.Column( - sqlalchemy.TIMESTAMP(timezone=True), - server_default=sqlalchemy.text("CURRENT_TIMESTAMP"), - ), - ) - updated_at: datetime.datetime = sqlmodel.Field( - default=None, - sa_column=sqlalchemy.Column( - sqlalchemy.TIMESTAMP(timezone=True), - server_default=sqlalchemy.text("CURRENT_TIMESTAMP"), - ), - ) - - _hydrated_content: HydratedContent | None = PrivateAttr(default=None) - _hydrated_content_source: tuple[StorageID | None, str] | None = PrivateAttr(default=None) - - async def get_hydrated_content(self, *, refresh: bool = False) -> HydratedContent: - """Return inline content or bytes loaded through the configured storage. - - The cache belongs only to this block instance. It is keyed by the persisted - storage pointer fields and is replaced when ``refresh`` is requested. - """ - source = (self.storage, self.content) - private_state = self.__pydantic_private__ - if private_state is None: - private_state = {} - object.__setattr__(self, "__pydantic_private__", private_state) - if ( - not refresh - and private_state.get("_hydrated_content_source") == source - and private_state.get("_hydrated_content") is not None - ): - return typing.cast(HydratedContent, private_state["_hydrated_content"]) - - if self.storage is None: - hydrated_content: HydratedContent = self.content - else: - from app.business.info_base.storage import StorageManager - - storage = await StorageManager.get_storage_async(self.storage) - stored_content = await storage.get_raw_content(self.content) - if not isinstance(stored_content, bytes): - raise TypeError( - f"Storage {storage.__class__.__name__} returned " - f"{type(stored_content).__name__}; configured storage must return bytes" - ) - hydrated_content = stored_content - - private_state["_hydrated_content"] = hydrated_content - private_state["_hydrated_content_source"] = source - return hydrated_content - - async def get_context_as_text(self) -> str | None: - from app.business.info_base.resolver import ResolverManager - - resolver = ResolverManager.get(self) - return await resolver.get_text() +if TYPE_CHECKING: + from inkcre_core.schemas.info_base.block import * diff --git a/app/schemas/info_base/main.py b/app/schemas/info_base/main.py index 8bd4b75e..a2f6c7fb 100644 --- a/app/schemas/info_base/main.py +++ b/app/schemas/info_base/main.py @@ -1,116 +1,10 @@ -"""Producer graph forms and graph-command results.""" +"""Legacy import; use inkcre_core for new consumers.""" -__all__ = [ - "Vector", - "OutArcForm", - "InArcForm", - "StarsGraphForm", - "GraphBlockForm", - "GraphRelationForm", - "GraphForm", - "GraphBlockIDMapping", - "SubmitGraphResult", -] +from typing import TYPE_CHECKING +from importlib import import_module +import sys -import typing +sys.modules[__name__] = import_module("inkcre_core.schemas.info_base.main") -import pydantic -import sqlmodel - -from app.schemas.info_base.block import BlockForm, BlockID -from app.schemas.info_base.relation import RelationForm - - -Vector: typing.TypeAlias = tuple[float, ...] -NegativeBlockID: typing.TypeAlias = typing.Annotated[int, pydantic.Field(lt=0)] -NonZeroBlockID: typing.TypeAlias = int - - -class OutArcForm(sqlmodel.SQLModel): - """One outgoing relation from the enclosing star to a recursive star.""" - - model_config = {"extra": "forbid"} - - relation: RelationForm - to_graph: "StarsGraphForm" - - -class InArcForm(sqlmodel.SQLModel): - """One incoming relation from a recursive star to the enclosing star.""" - - model_config = {"extra": "forbid"} - - relation: RelationForm - from_graph: "StarsGraphForm" - - -class StarsGraphForm(sqlmodel.SQLModel): - """Recursive star-centered authoring form used by Resolvers and extensions.""" - - model_config = {"extra": "forbid"} - - block: BlockForm - out_arcs: tuple[OutArcForm, ...] = () - in_arcs: tuple[InArcForm, ...] = () - - -class GraphBlockForm(BlockForm): - """A new Block declaration under one GraphForm-local negative ID.""" - - id: NegativeBlockID - - -class GraphRelationForm(RelationForm): - """A Relation declaration over the GraphForm signed Block-ID namespace.""" - - from_: NonZeroBlockID - to_: NonZeroBlockID - - @pydantic.field_validator("from_", "to_") - @classmethod - def require_non_zero_endpoint(cls, value: int) -> int: - if value == 0: - raise ValueError("GraphForm Relation endpoints must be non-zero") - return value - - -class GraphForm(sqlmodel.SQLModel): - """Flat command for adding arbitrarily connected Blocks and Relations.""" - - model_config = {"extra": "forbid"} - - blocks: tuple[GraphBlockForm, ...] = () - relations: tuple[GraphRelationForm, ...] = () - - @pydantic.model_validator(mode="after") - def validate_local_block_references(self) -> "GraphForm": - local_ids = [block.id for block in self.blocks] - if len(local_ids) != len(set(local_ids)): - raise ValueError("GraphForm new Block IDs must be unique") - - unresolved = { - endpoint - for relation in self.relations - for endpoint in (relation.from_, relation.to_) - if endpoint < 0 and endpoint not in local_ids - } - if unresolved: - rendered = ", ".join(str(value) for value in sorted(unresolved)) - raise ValueError(f"GraphForm contains unresolved local Block IDs: {rendered}") - return self - - -class GraphBlockIDMapping(sqlmodel.SQLModel): - """Map one command-local Block identity to its persisted identity.""" - - local_id: NegativeBlockID - id: BlockID - - -class SubmitGraphResult(sqlmodel.SQLModel): - """Minimal successful graph-command result used by producers and Agent Tools.""" - - blocks: tuple[GraphBlockIDMapping, ...] - - -StarsGraphForm.model_rebuild() +if TYPE_CHECKING: + from inkcre_core.schemas.info_base.main import * diff --git a/app/schemas/info_base/relation.py b/app/schemas/info_base/relation.py index 6712dba7..0b219e9c 100644 --- a/app/schemas/info_base/relation.py +++ b/app/schemas/info_base/relation.py @@ -1,68 +1,10 @@ -import typing -from typing import Optional as Opt -import datetime -import sqlalchemy -import sqlmodel +"""Legacy import; use inkcre_core for new consumers.""" -from app.schemas.info_base.block import BlockID +from typing import TYPE_CHECKING +from importlib import import_module +import sys -RelationID: typing.TypeAlias = int +sys.modules[__name__] = import_module("inkcre_core.schemas.info_base.relation") - -class RelationForm(sqlmodel.SQLModel): - """Producer-owned values for creating one Relation.""" - - model_config = {"extra": "forbid"} - - content: str = sqlmodel.Field( - sa_column=sqlalchemy.Column(sqlalchemy.Text, nullable=False) - ) - - -class RelationCreateForm(RelationForm): - """Standalone Relation creation command with persisted Block endpoints.""" - - from_: BlockID - to_: BlockID - - -class RelationModel(RelationForm, table=True): - __tablename__ = "relations" # type: ignore - __table_args__ = ( - sqlalchemy.Index( - "relations_from_id_desc_idx", - "from_", - sqlalchemy.desc("id"), - ), - sqlalchemy.Index( - "relations_to_id_desc_idx", - "to_", - sqlalchemy.desc("id"), - ), - ) - - id: Opt[RelationID] = sqlmodel.Field( - sa_column=sqlmodel.Column(sqlmodel.Integer, primary_key=True, autoincrement=True), - default=None, - ) - updated_at: datetime.datetime = sqlmodel.Field( - default=None, - sa_column=sqlalchemy.Column( - sqlalchemy.TIMESTAMP(timezone=True), - server_default=sqlalchemy.text("CURRENT_TIMESTAMP"), - ), - ) - from_: int = sqlmodel.Field( - sa_column=sqlalchemy.Column( - sqlalchemy.Integer, - sqlalchemy.ForeignKey("blocks.id", ondelete="CASCADE", onupdate="CASCADE"), - ), - default=0, - ) - to_: int = sqlmodel.Field( - sa_column=sqlalchemy.Column( - sqlalchemy.Integer, - sqlalchemy.ForeignKey("blocks.id", ondelete="CASCADE", onupdate="CASCADE"), - ), - default=0, - ) +if TYPE_CHECKING: + from inkcre_core.schemas.info_base.relation import * diff --git a/app/schemas/info_base/rest.py b/app/schemas/info_base/rest.py index 8981040e..9008a042 100644 --- a/app/schemas/info_base/rest.py +++ b/app/schemas/info_base/rest.py @@ -1,60 +1,10 @@ -"""Ordinary REST input forms; persisted entities and Peer wire formats stay separate.""" +"""Legacy import; use inkcre_core for new consumers.""" -import typing +from typing import TYPE_CHECKING +from importlib import import_module +import sys -import pydantic +sys.modules[__name__] = import_module("inkcre_core.schemas.info_base.rest") -from app.schemas.graph_navigation_retrieval import ( - DEFAULT_MAX_HOPS, - MAX_MAX_HOPS, - DEFAULT_MAX_EXPLORED_BLOCKS, - MAX_MAX_EXPLORED_BLOCKS, - DEFAULT_MAX_EXPLORED_RELATIONS, - GraphDirection, -) - - -class EntityReference(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid") - type: typing.Literal["block", "relation"] - id: int - - -class EntitiesGetForm(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid") - entities: list[EntityReference] - content: typing.Literal["none", "raw", "hydrated"] = "raw" - - -class BlockUpdateForm(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid") - content: str = pydantic.Field(default=None) # pyrefly: ignore[bad-assignment] - resolver: str = pydantic.Field(default=None) # pyrefly: ignore[bad-assignment] - storage: int | None = None - - -class RelationUpdateForm(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid") - from_block_id: int = pydantic.Field(default=None) # pyrefly: ignore[bad-assignment] - to_block_id: int = pydantic.Field(default=None) # pyrefly: ignore[bad-assignment] - content: str = pydantic.Field(default=None) # pyrefly: ignore[bad-assignment] - - -class GraphPathForm(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid") - from_block_id: int - to_block_id: int - direction: GraphDirection = "both" - contents: tuple[str, ...] = () - max_hops: int = pydantic.Field(DEFAULT_MAX_HOPS, ge=0, le=MAX_MAX_HOPS) - max_explored_blocks: int = pydantic.Field( - DEFAULT_MAX_EXPLORED_BLOCKS, ge=1, le=MAX_MAX_EXPLORED_BLOCKS - ) - - -class GraphComponentsForm(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid") - seed_block_ids: tuple[int, ...] - contents: tuple[str, ...] = pydantic.Field(min_length=1) - max_explored_blocks: int = pydantic.Field(DEFAULT_MAX_EXPLORED_BLOCKS, ge=1) - max_explored_relations: int = pydantic.Field(DEFAULT_MAX_EXPLORED_RELATIONS, ge=1) +if TYPE_CHECKING: + from inkcre_core.schemas.info_base.rest import * diff --git a/app/schemas/info_base/storage.py b/app/schemas/info_base/storage.py index 36a281b0..01b17e4f 100644 --- a/app/schemas/info_base/storage.py +++ b/app/schemas/info_base/storage.py @@ -1,94 +1,10 @@ -import typing -import uuid -import sqlalchemy -import sqlalchemy.dialects.postgresql -import sqlmodel -from typing import Optional as Opt +"""Legacy import; use inkcre_core for new consumers.""" +from typing import TYPE_CHECKING +from importlib import import_module +import sys -StorageTypeID: typing.TypeAlias = str -StorageID: typing.TypeAlias = int +sys.modules[__name__] = import_module("inkcre_core.schemas.info_base.storage") - -class StorageBlobModel(sqlmodel.SQLModel, table=True): - """Raw PostgreSQL-owned bytes addressed by an opaque generated pointer.""" - - __tablename__: str = "storage_blobs" # type: ignore - - id: uuid.UUID = sqlmodel.Field( - default_factory=uuid.uuid4, - sa_column=sqlalchemy.Column( - sqlalchemy.dialects.postgresql.UUID(as_uuid=True), - primary_key=True, - server_default=sqlalchemy.text("gen_random_uuid()"), - ), - ) - data: bytes = sqlmodel.Field( - sa_column=sqlalchemy.Column(sqlalchemy.LargeBinary, nullable=False) - ) - - -class StorageTypesModel(sqlmodel.SQLModel, table=True): - __tablename__: str = "storage_types" # type: ignore - - id: StorageTypeID = sqlmodel.Field( - sa_column=sqlalchemy.Column(sqlalchemy.Text, primary_key=True) - ) - """Type of storage. - - A custom string to identify the storage type. - For extension storages, must follow `extensions.{extension_id}.{type}` format. - """ - description: str = sqlmodel.Field( - sa_column=sqlalchemy.Column(sqlalchemy.Text, nullable=False) - ) - """Description of this storage type.""" - config_schema: dict = sqlmodel.Field( - sa_column=sqlalchemy.Column( - sqlalchemy.dialects.postgresql.JSONB, - server_default=sqlalchemy.text("'{}'::jsonb"), - nullable=False, - ), - default=dict, - ) - writable: bool = sqlmodel.Field( - default=False, - sa_column=sqlalchemy.Column( - sqlalchemy.Boolean, - nullable=False, - server_default=sqlalchemy.false(), - ), - ) - - -class StorageModel(sqlmodel.SQLModel, table=True): - __tablename__: str = "storages" # type: ignore - - id: Opt[StorageID] = sqlmodel.Field( - sa_column=sqlalchemy.Column(sqlalchemy.Integer, primary_key=True, autoincrement=True), - default=None, - ) - type: str = sqlmodel.Field( - sa_column=sqlalchemy.Column( - sqlalchemy.Text, - sqlalchemy.ForeignKey("storage_types.id", onupdate="CASCADE", ondelete="CASCADE"), - nullable=False, - ) - ) - """Type of storage. - - An absolute import path to the module where storage class at. - When delete a storage type, all storages of this type will be deleted too. - """ - nickname: Opt[str] = sqlmodel.Field( - default=None, - sa_column=sqlalchemy.Column(sqlalchemy.Text, nullable=True), - ) - config: dict = sqlmodel.Field( - sa_column=sqlalchemy.Column( - sqlalchemy.dialects.postgresql.JSONB, - server_default=sqlalchemy.text("'{}'::jsonb"), - nullable=False, - ), - default=dict, - ) +if TYPE_CHECKING: + from inkcre_core.schemas.info_base.storage import * diff --git a/app/schemas/job.py b/app/schemas/job.py index 740c0a88..41c93fea 100644 --- a/app/schemas/job.py +++ b/app/schemas/job.py @@ -1,173 +1,10 @@ -"""Durable background-command schemas.""" +"""Legacy import; use inkcre_core for new consumers.""" -import datetime -import enum -import typing +from typing import TYPE_CHECKING +from importlib import import_module +import sys -import pydantic -import sqlalchemy -import sqlalchemy.dialects.postgresql -import sqlmodel +sys.modules[__name__] = import_module("inkcre_core.schemas.job") - -JobTypeID: typing.TypeAlias = str -JobID: typing.TypeAlias = int - - -class JobStatus(enum.StrEnum): - PENDING = "pending" - RUNNING = "running" - FINISHED = "finished" - FAILED = "failed" - TIMED_OUT = "timed_out" - ABORTED = "aborted" - - @property - def terminal(self) -> bool: - return self not in {self.PENDING, self.RUNNING} - - -class JobTypeModel(sqlmodel.SQLModel, table=True): - """Database projection of one exact runtime Job Handler contract.""" - - __tablename__ = "job_types" # type: ignore - __table_args__ = ( - sqlalchemy.CheckConstraint( - "default_timeout_seconds > 0", - name="job_types_default_timeout_seconds_positive", - ), - ) - - id: JobTypeID = sqlmodel.Field( - sa_column=sqlalchemy.Column(sqlalchemy.Text, primary_key=True) - ) - description: str = sqlmodel.Field( - sa_column=sqlalchemy.Column(sqlalchemy.Text, nullable=False) - ) - parameters_schema: dict[str, typing.Any] = sqlmodel.Field( - default_factory=dict, - sa_column=sqlalchemy.Column( - sqlalchemy.dialects.postgresql.JSONB, - nullable=False, - server_default=sqlalchemy.text("'{}'::jsonb"), - ), - ) - default_timeout_seconds: int = sqlmodel.Field( - gt=0, - sa_column=sqlalchemy.Column(sqlalchemy.Integer, nullable=False), - ) - - -class JobModel(sqlmodel.SQLModel, table=True): - """One durable execution opportunity for an exact registered command.""" - - __tablename__ = "jobs" # type: ignore - __table_args__ = ( - sqlalchemy.Index("jobs_status_idx", "status"), - sqlalchemy.CheckConstraint( - "octet_length(submission_traceparent) <= 512", - name="jobs_submission_traceparent_capacity", - ), - sqlalchemy.CheckConstraint( - "octet_length(submission_tracestate) <= 512", - name="jobs_submission_tracestate_capacity", - ), - sqlalchemy.CheckConstraint( - "timeout_seconds > 0", - name="jobs_timeout_seconds_positive", - ), - sqlalchemy.CheckConstraint( - "(status = 'pending' AND started_at IS NULL AND closed_at IS NULL) OR " - "(status = 'running' AND started_at IS NOT NULL AND closed_at IS NULL) OR " - "(status IN ('finished', 'failed', 'timed_out', 'aborted') " - "AND closed_at IS NOT NULL)", - name="jobs_lifecycle_timestamps_valid", - ), - ) - - id: JobID | None = sqlmodel.Field( - default=None, - sa_column=sqlalchemy.Column( - sqlalchemy.BigInteger, - sqlalchemy.Identity(), - primary_key=True, - ), - ) - type: JobTypeID = sqlmodel.Field( - sa_column=sqlalchemy.Column( - sqlalchemy.Text, - sqlalchemy.ForeignKey("job_types.id", onupdate="CASCADE", ondelete="RESTRICT"), - nullable=False, - ) - ) - parameters: dict[str, typing.Any] = sqlmodel.Field( - default_factory=dict, - sa_column=sqlalchemy.Column( - sqlalchemy.dialects.postgresql.JSONB, - nullable=False, - server_default=sqlalchemy.text("'{}'::jsonb"), - ), - ) - submission_traceparent: str | None = sqlmodel.Field( - default=None, - sa_column=sqlalchemy.Column(sqlalchemy.Text, nullable=True), - ) - submission_tracestate: str | None = sqlmodel.Field( - default=None, - sa_column=sqlalchemy.Column(sqlalchemy.Text, nullable=True), - ) - state: dict[str, typing.Any] = sqlmodel.Field( - default_factory=dict, - sa_column=sqlalchemy.Column( - sqlalchemy.dialects.postgresql.JSONB, - nullable=False, - server_default=sqlalchemy.text("'{}'::jsonb"), - ), - ) - abort_requested: bool = sqlmodel.Field( - default=False, - sa_column=sqlalchemy.Column( - sqlalchemy.Boolean, nullable=False, server_default=sqlalchemy.false() - ), - ) - timeout_seconds: int = sqlmodel.Field( - gt=0, - sa_column=sqlalchemy.Column(sqlalchemy.Integer, nullable=False), - ) - status: JobStatus = sqlmodel.Field( - default=JobStatus.PENDING, - sa_column=sqlalchemy.Column( - sqlalchemy.Enum( - JobStatus, - name="jobstatus", - values_callable=lambda values: [value.value for value in values], - inherit_schema=True, - ), - nullable=False, - server_default=JobStatus.PENDING.value, - ), - ) - created_at: datetime.datetime = sqlmodel.Field( - default=None, - sa_column=sqlalchemy.Column( - sqlalchemy.TIMESTAMP(timezone=True), - nullable=False, - server_default=sqlalchemy.text("CURRENT_TIMESTAMP"), - ), - ) - started_at: datetime.datetime | None = sqlmodel.Field( - default=None, - sa_column=sqlalchemy.Column(sqlalchemy.TIMESTAMP(timezone=True), nullable=True), - ) - closed_at: datetime.datetime | None = sqlmodel.Field( - default=None, - sa_column=sqlalchemy.Column(sqlalchemy.TIMESTAMP(timezone=True), nullable=True), - ) - - -class JobCreateForm(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid") - - type: JobTypeID - parameters: dict[str, typing.Any] = pydantic.Field(default_factory=dict) - timeout_seconds: int | None = pydantic.Field(default=None, gt=0) +if TYPE_CHECKING: + from inkcre_core.schemas.job import * diff --git a/app/schemas/lexical_retrieval.py b/app/schemas/lexical_retrieval.py index 60b6576b..a045fc64 100644 --- a/app/schemas/lexical_retrieval.py +++ b/app/schemas/lexical_retrieval.py @@ -1,150 +1,10 @@ -"""Block-local lexical projection, maintenance, and ranked result contracts.""" +"""Legacy import; use inkcre_core for new consumers.""" -import datetime -import math -import typing +from typing import TYPE_CHECKING +from importlib import import_module +import sys -import pydantic -import sqlalchemy -import sqlalchemy.dialects.postgresql -import sqlmodel +sys.modules[__name__] = import_module("inkcre_core.schemas.lexical_retrieval") -from app.schemas.info_base.block import BlockModel - - -LexicalEvidence: typing.TypeAlias = typing.Literal[ - "label_exact", - "label_substring", - "text_substring", - "terms", -] - - -class BlockLexicalRecordModel(sqlmodel.SQLModel, table=True): - """Rebuildable Resolver projection owned only by lexical retrieval.""" - - __tablename__ = "block_lexical_records" # type: ignore - __table_args__ = ( - sqlalchemy.Index( - "block_lexical_records_search_vector_idx", - "search_vector", - postgresql_using="gin", - ), - sqlalchemy.Index( - "block_lexical_records_label_trgm_idx", - "label", - postgresql_using="gin", - postgresql_ops={"label": "gin_trgm_ops"}, - ), - sqlalchemy.Index( - "block_lexical_records_text_trgm_idx", - "text", - postgresql_using="gin", - postgresql_ops={"text": "gin_trgm_ops"}, - ), - ) - - block: int = sqlmodel.Field( - sa_column=sqlalchemy.Column( - sqlalchemy.Integer, - sqlalchemy.ForeignKey("blocks.id", onupdate="CASCADE", ondelete="CASCADE"), - primary_key=True, - ) - ) - label: str = sqlmodel.Field(sa_column=sqlalchemy.Column(sqlalchemy.Text, nullable=False)) - text: str | None = sqlmodel.Field( - default=None, - sa_column=sqlalchemy.Column(sqlalchemy.Text, nullable=True), - ) - search_vector: str = sqlmodel.Field( - sa_column=sqlalchemy.Column( - sqlalchemy.dialects.postgresql.TSVECTOR, - nullable=False, - ) - ) - created_at: datetime.datetime = sqlmodel.Field( - default=None, - sa_column=sqlalchemy.Column( - sqlalchemy.TIMESTAMP(timezone=True), - nullable=False, - server_default=sqlalchemy.text("CURRENT_TIMESTAMP"), - ), - ) - updated_at: datetime.datetime = sqlmodel.Field( - default=None, - sa_column=sqlalchemy.Column( - sqlalchemy.TIMESTAMP(timezone=True), - nullable=False, - server_default=sqlalchemy.text("CURRENT_TIMESTAMP"), - ), - ) - - -class LexicalRetrievalRequest(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - query: str - limit: int = pydantic.Field(default=20, ge=1, le=20) - - @pydantic.field_validator("query") - @classmethod - def non_empty_query(cls, value: str) -> str: - if not value.strip(): - raise ValueError("query must not be empty") - return value - - -class LexicalRetrievalMatch(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - block: BlockModel - label: str - excerpt: str - evidence: LexicalEvidence - rank: float = pydantic.Field(ge=0) - - @pydantic.field_validator("rank") - @classmethod - def finite_rank(cls, value: float) -> float: - if not math.isfinite(value): - raise ValueError("rank must be finite") - return value - - -class LexicalRetrievalResult(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - matches: tuple[LexicalRetrievalMatch, ...] - - -class LexicalMaintenanceOptions(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - max_records: int = pydantic.Field(default=100, ge=1) - scan_page_size: int = pydantic.Field(default=100, ge=1) - diagnostic_limit: int = pydantic.Field(default=20, ge=0) - - -class LexicalMaintenanceDiagnostic(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - block: int - outcome: typing.Literal["unavailable", "failed"] - reason: str - - -class LexicalMaintenanceReport(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - indexed: int = 0 - unavailable: int = 0 - failed: int = 0 - diagnostics: tuple[LexicalMaintenanceDiagnostic, ...] = () - - -class LexicalMaintenanceJobParameters(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - options: LexicalMaintenanceOptions = pydantic.Field( - default_factory=LexicalMaintenanceOptions - ) +if TYPE_CHECKING: + from inkcre_core.schemas.lexical_retrieval import * diff --git a/app/schemas/observability.py b/app/schemas/observability.py index b8e588ad..f48b1ac8 100644 --- a/app/schemas/observability.py +++ b/app/schemas/observability.py @@ -1,37 +1,10 @@ -"""Non-secret deployment telemetry configuration shared by admitted Peers.""" +"""Legacy import; use inkcre_core for new consumers.""" -import uuid -import pydantic +from typing import TYPE_CHECKING +from importlib import import_module +import sys -CONFIG_KEY = "inkcre.observability" -CONFIG_SCHEMA = "inkcre.observability.v1" +sys.modules[__name__] = import_module("inkcre_core.schemas.observability") - -def _credential_free_url(value: pydantic.AnyHttpUrl | None): - if value is not None and ( - value.username or value.password or value.query or value.fragment - ): - raise ValueError("observability URLs cannot contain credentials, queries or fragments") - return value - - -class TelemetryEndpoints(pydantic.BaseModel): - """Complete OTLP/HTTP destinations; authentication stays in local runtime config.""" - - model_config = pydantic.ConfigDict(extra="forbid") - traces: pydantic.AnyHttpUrl | None = None - logs: pydantic.AnyHttpUrl | None = None - metrics: pydantic.AnyHttpUrl | None = None - - _urls = pydantic.field_validator("traces", "logs", "metrics")(_credential_free_url) - - -class ObservabilityConfig(pydantic.BaseModel): - """Stable deployment identity and optional metadata-only diagnostic destinations.""" - - model_config = pydantic.ConfigDict(extra="forbid") - deployment_id: uuid.UUID - otlp_http_endpoints: TelemetryEndpoints | None = None - diagnostics_url: pydantic.AnyHttpUrl | None = None - - _url = pydantic.field_validator("diagnostics_url")(_credential_free_url) +if TYPE_CHECKING: + from inkcre_core.schemas.observability import * diff --git a/app/schemas/organization.py b/app/schemas/organization.py index e1183482..afb363a0 100644 --- a/app/schemas/organization.py +++ b/app/schemas/organization.py @@ -1,89 +1,10 @@ -"""Organization rumination configuration, inbound, and Agent Tool forms.""" +"""Legacy import; use inkcre_core for new consumers.""" -import typing +from typing import TYPE_CHECKING +from importlib import import_module +import sys -import pydantic +sys.modules[__name__] = import_module("inkcre_core.schemas.organization") -from app.schemas.ai import JSONValue -from app.schemas.info_base.block import BlockID -from app.schemas.info_base.main import GraphForm, NegativeBlockID - - -class RuminationConfig(pydantic.BaseModel): - """Deployment selection of one reusable Agent definition.""" - - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - agent: int - - -class MediaInterpretationConfig(pydantic.BaseModel): - """Deployment selection of independent modality-oriented Agents.""" - - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - image_agent: int - audio_agent: int - video_agent: int - - -class MediaInterpretationJobParameters(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - -class MediaInterpretationDiagnostic(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - block: int - modality: typing.Literal["image", "audio", "video"] - outcome: typing.Literal["unavailable", "failed", "no_output"] - reason: str - - -class MediaInterpretationReport(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - selected: int = 0 - interpreted: int = 0 - unavailable: int = 0 - failed: int = 0 - no_output: int = 0 - diagnostics: tuple[MediaInterpretationDiagnostic, ...] = () - - -class RuminationRequest(pydantic.BaseModel): - """Explicit request to reconsider one focal Block.""" - - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - block: BlockID - - -class GetDraftGraphSchemaInput(pydantic.BaseModel): - """Generic schema-discovery input narrowed when an Agent run binds Tools.""" - - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - resolver_types: tuple[str, ...] - - -class DraftGraphInput(pydantic.BaseModel): - """Generic draft request; Resolver input is validated by its bound run model.""" - - model_config = pydantic.ConfigDict(extra="forbid") - - resolver_type: str - input: dict[str, JSONValue] = pydantic.Field( - description="Arguments matching the selected Resolver's input_schema." - ) - local_block_id_start: NegativeBlockID = pydantic.Field( - default=-1, description="First temporary ID; keep IDs disjoint when combining drafts." - ) - - -class SubmitGraphInput(pydantic.BaseModel): - """The sole rumination Tool input that can mutate the info-base graph.""" - - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - graph: GraphForm +if TYPE_CHECKING: + from inkcre_core.schemas.organization import * diff --git a/app/schemas/organization_behavior.py b/app/schemas/organization_behavior.py index 43915bf9..2bbcae1c 100644 --- a/app/schemas/organization_behavior.py +++ b/app/schemas/organization_behavior.py @@ -1,146 +1,10 @@ -"""Contracts for exact Organization behaviors and their Agent adapters.""" +"""Legacy import; use inkcre_core for new consumers.""" -import typing +from typing import TYPE_CHECKING +from importlib import import_module +import sys -import pydantic +sys.modules[__name__] = import_module("inkcre_core.schemas.organization_behavior") -from app.schemas.graph_navigation_retrieval import ( - GraphModel, -) -from app.schemas.info_base.block import BlockID, ResolverType -from app.schemas.info_base.relation import RelationID - - -class BehaviorAgentConfig(pydantic.BaseModel): - """Deployment selection of one purpose-built Agent definition.""" - - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - agent: int - - -class AutomaticOrganizationJobParameters(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - max_seeds: int = pydantic.Field(default=10, ge=3, le=100) - - -class RelationWriteResult(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - relation_id: RelationID - created: bool - - -class CandidateWriteResult(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - descriptor_block_id: BlockID - relation_id: RelationID - created: bool - - -class SupersessionProposal(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - successor_block_id: BlockID - predecessor_block_id: BlockID - - -class RefinementProposal(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - refinement_block_id: BlockID - predecessor_block_id: BlockID - - -class EvidenceStanceProposal(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - evidence_block_id: BlockID - assertion_block_id: BlockID - stance: typing.Literal["supports", "challenges"] - - -class SynthesisProposal(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - text: str - source_block_ids: tuple[BlockID, ...] = pydantic.Field( - min_length=2, description="Actual contributing sources, not all inspected context." - ) - previous_synthesis_block_id: BlockID | None = pydantic.Field( - default=None, description="Earlier synthesis revised through an edited relation." - ) - - @pydantic.field_validator("text") - @classmethod - def non_empty_text(cls, value: str) -> str: - if not value.strip(): - raise ValueError("text must not be empty") - return value - - @pydantic.field_validator("source_block_ids") - @classmethod - def distinct_sources(cls, value: tuple[BlockID, ...]) -> tuple[BlockID, ...]: - if len(set(value)) != len(value): - raise ValueError("source_block_ids must be distinct") - return value - - -class SynthesisWriteResult(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - synthesis_block_id: BlockID - synthesis_created: bool - basis: tuple[RelationWriteResult, ...] - edited: RelationWriteResult | None = None - - -class ExistingReferentAnchorProposal(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - source_block_id: BlockID - selected_text: str = pydantic.Field( - description="Minimal sufficient referring fragment from the source." - ) - referent_block_id: BlockID - - @pydantic.field_validator("selected_text") - @classmethod - def non_empty_selected_text(cls, value: str) -> str: - if not value.strip(): - raise ValueError("selected_text must not be empty") - return value - - -class ExistingReferentAnchorResult(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - fragment_block_id: BlockID - fragment_created: bool - has_mention: RelationWriteResult | None - refers_to: RelationWriteResult - - -class DuplicateAssertionProposal(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - left_block_id: BlockID - right_block_id: BlockID - - -class RecordOrganizationCandidateInput(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - block_id: BlockID - behavior: ResolverType - - -class SupersessionLineage(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - graph: GraphModel - current_block_ids: tuple[BlockID, ...] - truncated: bool - cycle_detected: bool +if TYPE_CHECKING: + from inkcre_core.schemas.organization_behavior import * diff --git a/app/schemas/peer/__init__.py b/app/schemas/peer/__init__.py index 6c1618f4..f0937bae 100644 --- a/app/schemas/peer/__init__.py +++ b/app/schemas/peer/__init__.py @@ -1,35 +1,3 @@ -from .main import ( - CapabilityID, - CorePeerConfig, - PeerCapabilityAdvertisement, - PeerInboundInterface, - PeerModel, - PeerProtocolID, - PeerRef, - normalize_capability_snapshot, -) -from .protocol import ( - PEER_EXECUTION_HEADER, - PEER_HTTP_PROTOCOL, - PEER_NOT_EXECUTED, - PeerHTTPInboundParameters, - PeerProtocolRequest, - PeerProtocolResponse, -) +"""Legacy import; use inkcre_core for new consumers.""" -__all__ = [ - "CapabilityID", - "CorePeerConfig", - "PEER_EXECUTION_HEADER", - "PEER_HTTP_PROTOCOL", - "PEER_NOT_EXECUTED", - "PeerCapabilityAdvertisement", - "PeerHTTPInboundParameters", - "PeerInboundInterface", - "PeerModel", - "PeerProtocolID", - "PeerProtocolRequest", - "PeerProtocolResponse", - "PeerRef", - "normalize_capability_snapshot", -] +from inkcre_core.schemas.peer import * diff --git a/app/schemas/peer/main.py b/app/schemas/peer/main.py index 060fd7e2..7822de78 100644 --- a/app/schemas/peer/main.py +++ b/app/schemas/peer/main.py @@ -1,178 +1,10 @@ -"""Peer identity, configuration, capability advertisement, and persistence.""" +"""Legacy import; use inkcre_core for new consumers.""" -import datetime -import typing -from urllib.parse import urlsplit, urlunsplit -import uuid +from typing import TYPE_CHECKING +from importlib import import_module +import sys -import pydantic -import sqlalchemy -import sqlalchemy.dialects.postgresql -import sqlmodel +sys.modules[__name__] = import_module("inkcre_core.schemas.peer.main") -from app.schemas.ai import JSONValue - - -PeerRef: typing.TypeAlias = uuid.UUID -CapabilityID: typing.TypeAlias = str -PeerProtocolID: typing.TypeAlias = str - - -class PeerInboundInterface(pydantic.BaseModel): - """One protocol-discriminated construction descriptor for a caller outbound.""" - - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - protocol: PeerProtocolID = pydantic.Field(min_length=1) - parameters: dict[str, JSONValue] - - -class PeerCapabilityAdvertisement(pydantic.BaseModel): - """One exact capability and its provider-owned inbound interface.""" - - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - id: CapabilityID = pydantic.Field(min_length=1) - inbound: PeerInboundInterface - - -_CAPABILITIES_ADAPTER = pydantic.TypeAdapter(tuple[PeerCapabilityAdvertisement, ...]) - - -def normalize_capability_snapshot( - value: typing.Any, -) -> tuple[PeerCapabilityAdvertisement, ...]: - """Validate, de-duplicate, and deterministically order one full snapshot.""" - capabilities = _CAPABILITIES_ADAPTER.validate_python(value or ()) - ids = tuple(capability.id for capability in capabilities) - if len(ids) != len(set(ids)): - raise ValueError("Peer capability IDs must be unique") - return tuple(sorted(capabilities, key=lambda capability: capability.id)) - - -class CorePeerConfig(pydantic.BaseModel): - """impl-py-owned values stored in this Peer row's config object.""" - - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - http_public_base_url: str | None = pydantic.Field(default=None, title="Public HTTP URL") - extension_registry_url: str | None = pydantic.Field( - default=None, title="Extension Registry URL" - ) - - @pydantic.field_validator("http_public_base_url") - @classmethod - def valid_public_http_base(cls, value: str | None) -> str | None: - if value is None: - return None - parts = urlsplit(value) - if ( - parts.scheme not in {"http", "https"} - or not parts.netloc - or parts.username is not None - or parts.password is not None - or parts.query - or parts.fragment - ): - raise ValueError( - "http_public_base_url must be an absolute HTTP(S) URL without " - "credentials, query, or fragment" - ) - path = parts.path.rstrip("/") - return urlunsplit((parts.scheme, parts.netloc, path, "", "")) - - @pydantic.field_validator("extension_registry_url") - @classmethod - def valid_extension_registry_origin(cls, value: str | None) -> str | None: - if value is None or not value.strip(): - return None - parts = urlsplit(value.strip()) - if ( - parts.scheme not in {"http", "https"} - or not parts.netloc - or parts.username is not None - or parts.password is not None - or parts.path not in {"", "/"} - or parts.query - or parts.fragment - ): - raise ValueError("extension_registry_url must be one HTTP(S) origin") - return urlunsplit((parts.scheme, parts.netloc, "", "", "")) - - -class PeerModel(sqlmodel.SQLModel, table=True): - """One equal deployment Peer with a runtime-owned capability/lease snapshot.""" - - __tablename__ = "peers" # type: ignore - - id: PeerRef = sqlmodel.Field( - sa_column=sqlalchemy.Column( - sqlalchemy.dialects.postgresql.UUID(as_uuid=True), - primary_key=True, - default=uuid.uuid4, - ), - ) - name: str = sqlmodel.Field(sa_column=sqlalchemy.Column(sqlalchemy.Text, nullable=False)) - application_version: str | None = sqlmodel.Field( - default=None, - sa_column=sqlalchemy.Column(sqlalchemy.Text, nullable=True), - ) - labels: list[str] = sqlmodel.Field( - default_factory=list, - sa_column=sqlalchemy.Column( - sqlalchemy.dialects.postgresql.ARRAY(sqlalchemy.Text), - nullable=False, - server_default=sqlalchemy.text("'{}'::text[]"), - ), - ) - config: dict[str, JSONValue] = sqlmodel.Field( - default_factory=dict, - sa_column=sqlalchemy.Column( - sqlalchemy.dialects.postgresql.JSONB, - nullable=False, - server_default=sqlalchemy.text("'{}'::jsonb"), - ), - ) - config_schema: dict[str, JSONValue] = sqlmodel.Field( - default_factory=dict, - sa_column=sqlalchemy.Column( - sqlalchemy.dialects.postgresql.JSONB, - nullable=False, - server_default=sqlalchemy.text("'{}'::jsonb"), - ), - ) - capabilities: list[dict[str, JSONValue]] = sqlmodel.Field( - default_factory=list, - sa_column=sqlalchemy.Column( - sqlalchemy.dialects.postgresql.JSONB, - nullable=False, - server_default=sqlalchemy.text("'[]'::jsonb"), - ), - ) - lease_expires_at: datetime.datetime | None = sqlmodel.Field( - default=None, - sa_column=sqlalchemy.Column( - sqlalchemy.TIMESTAMP(timezone=True), - nullable=True, - ), - ) - created_at: datetime.datetime = sqlmodel.Field( - default=None, - sa_column=sqlalchemy.Column( - sqlalchemy.TIMESTAMP(timezone=True), - nullable=False, - server_default=sqlalchemy.text("CURRENT_TIMESTAMP"), - ), - ) - updated_at: datetime.datetime = sqlmodel.Field( - default=None, - sa_column=sqlalchemy.Column( - sqlalchemy.TIMESTAMP(timezone=True), - nullable=False, - server_default=sqlalchemy.text("CURRENT_TIMESTAMP"), - ), - ) - - def capability_snapshot(self) -> tuple[PeerCapabilityAdvertisement, ...]: - """Validate a possibly externally edited persisted snapshot at use time.""" - return _CAPABILITIES_ADAPTER.validate_python(self.capabilities or ()) +if TYPE_CHECKING: + from inkcre_core.schemas.peer.main import * diff --git a/app/schemas/peer/protocol.py b/app/schemas/peer/protocol.py index 38f40e12..874ceadf 100644 --- a/app/schemas/peer/protocol.py +++ b/app/schemas/peer/protocol.py @@ -1,99 +1,10 @@ -"""Normalized Peer HTTP v1 request/response envelopes.""" +"""Legacy import; use inkcre_core for new consumers.""" -import re -import typing -from urllib.parse import urlsplit +from typing import TYPE_CHECKING +from importlib import import_module +import sys -import pydantic +sys.modules[__name__] = import_module("inkcre_core.schemas.peer.protocol") -from app.schemas.ai import JSONValue - - -PEER_HTTP_PROTOCOL = "core.peer.protocol.http.v1" -PEER_EXECUTION_HEADER = "InkCre-Peer-Execution" -PEER_NOT_EXECUTED = "not-executed" - -_HTTP_TOKEN = re.compile(r"^[!#$%&'*+.^_`|~0-9A-Za-z-]+$") - - -def _normalized_multimap(value: typing.Any) -> dict[str, tuple[str, ...]]: - if value is None: - return {} - if not isinstance(value, dict): - raise ValueError("normalized map must be an object") - normalized: dict[str, tuple[str, ...]] = {} - for raw_name, raw_values in value.items(): - if not isinstance(raw_name, str) or not raw_name: - raise ValueError("normalized map names must be non-empty strings") - name = raw_name.lower() - if name in normalized: - raise ValueError(f"normalized map contains duplicate name: {name}") - if not isinstance(raw_values, (list, tuple)) or not all( - isinstance(item, str) for item in raw_values - ): - raise ValueError("normalized map values must be string arrays") - normalized[name] = tuple(raw_values) - return normalized - - -class PeerProtocolRequest(pydantic.BaseModel): - """One protocol payload; query, headers, and JSON body may coexist.""" - - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - query: dict[str, tuple[str, ...]] = pydantic.Field(default_factory=dict) - headers: dict[str, tuple[str, ...]] = pydantic.Field(default_factory=dict) - body: JSONValue = None - - @pydantic.field_validator("query", "headers", mode="before") - @classmethod - def normalized_maps(cls, value: typing.Any) -> dict[str, tuple[str, ...]]: - return _normalized_multimap(value) - - -class PeerProtocolResponse(pydantic.BaseModel): - """One executed HTTP response projected back into the normalized protocol.""" - - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - status: int = pydantic.Field(ge=100, le=599) - headers: dict[str, tuple[str, ...]] = pydantic.Field(default_factory=dict) - body: JSONValue = None - - @pydantic.field_validator("headers", mode="before") - @classmethod - def normalized_headers(cls, value: typing.Any) -> dict[str, tuple[str, ...]]: - return _normalized_multimap(value) - - -class PeerHTTPInboundParameters(pydantic.BaseModel): - """Static parameters published by one HTTP business inbound.""" - - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - method: str - url: str - - @pydantic.field_validator("method") - @classmethod - def valid_method(cls, value: str) -> str: - method = value.upper() - if not _HTTP_TOKEN.fullmatch(method): - raise ValueError("HTTP method must be one valid token") - return method - - @pydantic.field_validator("url") - @classmethod - def valid_absolute_url(cls, value: str) -> str: - parts = urlsplit(value) - if ( - parts.scheme not in {"http", "https"} - or not parts.netloc - or parts.username is not None - or parts.password is not None - or parts.fragment - ): - raise ValueError( - "Peer HTTP inbound URL must be absolute HTTP(S) without credentials or fragment" - ) - return value +if TYPE_CHECKING: + from inkcre_core.schemas.peer.protocol import * diff --git a/app/schemas/semantic_retrieval.py b/app/schemas/semantic_retrieval.py index 5b59e868..ab06b679 100644 --- a/app/schemas/semantic_retrieval.py +++ b/app/schemas/semantic_retrieval.py @@ -1,129 +1,10 @@ -"""Semantic retrieval configuration, maintenance and ranked result contracts.""" +"""Legacy import; use inkcre_core for new consumers.""" -import math -import typing +from typing import TYPE_CHECKING +from importlib import import_module +import sys -import pydantic +sys.modules[__name__] = import_module("inkcre_core.schemas.semantic_retrieval") -from app.schemas.ai import EmbeddingProfileID -from app.schemas.info_base.block import BlockModel -from app.schemas.info_base.relation import RelationModel - - -SemanticEntityType: typing.TypeAlias = typing.Literal["block", "relation"] - - -def _default_entity_types() -> frozenset[SemanticEntityType]: - return frozenset(("block", "relation")) - - -class SemanticRetrievalConfig(pydantic.BaseModel): - """Deployment-owned selection of the default vector space.""" - - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - default_profile: EmbeddingProfileID | None = None - - -class VectorRetrievalOptions(pydantic.BaseModel): - """Query-scoped controls over one already compatible vector space.""" - - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - limit: int = pydantic.Field(default=20, ge=1, le=100) - min_score: float | None = pydantic.Field(default=None, ge=-1, le=1) - entity_types: frozenset[SemanticEntityType] = pydantic.Field( - default_factory=_default_entity_types, - min_length=1, - ) - - @pydantic.field_validator("min_score") - @classmethod - def finite_min_score(cls, value: float | None) -> float | None: - if value is not None and not math.isfinite(value): - raise ValueError("min_score must be finite") - return value - - -class SemanticRetrievalRequest(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - query: str - profile: EmbeddingProfileID | None = None - options: VectorRetrievalOptions = pydantic.Field(default_factory=VectorRetrievalOptions) - - @pydantic.field_validator("query") - @classmethod - def non_empty_query(cls, value: str) -> str: - if not value.strip(): - raise ValueError("query must not be empty") - return value - - -class BlockSemanticRetrievalMatch(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - type: typing.Literal["block"] = "block" - entity: BlockModel - score: float = pydantic.Field(ge=-1, le=1) - - -class RelationSemanticRetrievalMatch(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - type: typing.Literal["relation"] = "relation" - entity: RelationModel - score: float = pydantic.Field(ge=-1, le=1) - - -SemanticRetrievalMatch: typing.TypeAlias = typing.Annotated[ - BlockSemanticRetrievalMatch | RelationSemanticRetrievalMatch, - pydantic.Field(discriminator="type"), -] - - -class SemanticRetrievalResult(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - profile: EmbeddingProfileID - metric: typing.Literal["cosine"] = "cosine" - matches: tuple[SemanticRetrievalMatch, ...] - - -class EmbeddingMaintenanceOptions(pydantic.BaseModel): - """Peer-local bounds for one resumable maintenance invocation.""" - - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - max_embeddings: int = pydantic.Field(default=100, ge=1) - batch_size: int = pydantic.Field(default=20, ge=1) - scan_page_size: int = pydantic.Field(default=100, ge=1) - diagnostic_limit: int = pydantic.Field(default=20, ge=0) - - -class EmbeddingMaintenanceDiagnostic(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - entity_type: SemanticEntityType - entity: int - outcome: typing.Literal["unavailable", "failed"] - reason: str - - -class EmbeddingMaintenanceReport(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - profile: EmbeddingProfileID - embedded: int = 0 - unavailable: int = 0 - failed: int = 0 - diagnostics: tuple[EmbeddingMaintenanceDiagnostic, ...] = () - - -class EmbeddingMaintenanceJobParameters(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - profile: EmbeddingProfileID | None = None - options: EmbeddingMaintenanceOptions = pydantic.Field( - default_factory=EmbeddingMaintenanceOptions - ) +if TYPE_CHECKING: + from inkcre_core.schemas.semantic_retrieval import * diff --git a/app/schemas/sink.py b/app/schemas/sink.py index ff3739be..cdaa0c34 100644 --- a/app/schemas/sink.py +++ b/app/schemas/sink.py @@ -1,82 +1,10 @@ -"""Persisted Sink catalog and instance contracts.""" +"""Legacy import; use inkcre_core for new consumers.""" -import typing -import uuid +from typing import TYPE_CHECKING +from importlib import import_module +import sys -import pydantic -import sqlalchemy -import sqlalchemy.dialects.postgresql -import sqlmodel +sys.modules[__name__] = import_module("inkcre_core.schemas.sink") - -SinkTypeID: typing.TypeAlias = str -SinkID: typing.TypeAlias = int - - -class SinkTypeModel(sqlmodel.SQLModel, table=True): - __tablename__ = "sink_types" # type: ignore - - id: SinkTypeID = sqlmodel.Field( - sa_column=sqlalchemy.Column(sqlalchemy.Text, primary_key=True) - ) - description: str = sqlmodel.Field( - sa_column=sqlalchemy.Column(sqlalchemy.Text, nullable=False) - ) - config_schema: dict[str, typing.Any] = sqlmodel.Field( - default_factory=dict, - sa_column=sqlalchemy.Column( - sqlalchemy.dialects.postgresql.JSONB, - nullable=False, - server_default=sqlalchemy.text("'{}'::jsonb"), - ), - ) - - -class SinkModel(sqlmodel.SQLModel, table=True): - __tablename__ = "sinks" # type: ignore - - id: SinkID | None = sqlmodel.Field( - default=None, - sa_column=sqlalchemy.Column( - sqlalchemy.BigInteger, - sqlalchemy.Identity(), - primary_key=True, - ), - ) - type: SinkTypeID = sqlmodel.Field( - sa_column=sqlalchemy.Column( - sqlalchemy.Text, - sqlalchemy.ForeignKey("sink_types.id", onupdate="CASCADE", ondelete="RESTRICT"), - nullable=False, - ) - ) - nickname: str | None = sqlmodel.Field( - default=None, - sa_column=sqlalchemy.Column(sqlalchemy.Text, nullable=True), - ) - config: dict[str, typing.Any] = sqlmodel.Field( - default_factory=dict, - sa_column=sqlalchemy.Column( - sqlalchemy.dialects.postgresql.JSONB, - nullable=False, - server_default=sqlalchemy.text("'{}'::jsonb"), - ), - ) - enabled: list[uuid.UUID] = sqlmodel.Field( - default_factory=list, - sa_column=sqlalchemy.Column( - sqlalchemy.dialects.postgresql.ARRAY( - sqlalchemy.dialects.postgresql.UUID(as_uuid=True) - ), - nullable=False, - server_default=sqlalchemy.text("'{}'::uuid[]"), - ), - ) - - -class SinkCreateForm(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid", frozen=True) - - type: SinkTypeID - nickname: str | None = None - config: dict[str, typing.Any] = pydantic.Field(default_factory=dict) +if TYPE_CHECKING: + from inkcre_core.schemas.sink import * diff --git a/app/schemas/source/__init__.py b/app/schemas/source/__init__.py index a400ba16..4c184826 100644 --- a/app/schemas/source/__init__.py +++ b/app/schemas/source/__init__.py @@ -1,19 +1,3 @@ -__all__ = [ - "SourceID", - "SourceModel", - "SourceTypesModel", - "SourceCollectParameters", - "SourceBackfillParameters", - "SourceCreateForm", - "SourceUpdateForm", -] +"""Legacy import; use inkcre_core for new consumers.""" -from .main import ( - SourceBackfillParameters, - SourceCollectParameters, - SourceID, - SourceModel, - SourceTypesModel, - SourceCreateForm, - SourceUpdateForm, -) +from inkcre_core.schemas.source import * diff --git a/app/schemas/source/main.py b/app/schemas/source/main.py index 04792aaf..0022f741 100644 --- a/app/schemas/source/main.py +++ b/app/schemas/source/main.py @@ -1,186 +1,11 @@ -__all__ = [ - "SourceID", - "SourceModel", - "SourceTypesModel", - "SourceCollectParameters", - "SourceBackfillParameters", -] +"""Legacy import; use inkcre_core for new consumers.""" -import datetime -import pydantic -import sqlalchemy -import sqlalchemy.dialects -import sqlalchemy.dialects.postgresql -import sqlmodel -import typing -from typing import Optional as Opt +from typing import TYPE_CHECKING +from importlib import import_module +import sys -from app.schemas.info_base.block import BlockID -from app.schemas.info_base.storage import StorageID +sys.modules[__name__] = import_module("inkcre_core.schemas.source.main") -SourceID: typing.TypeAlias = int - - -class SourceCreateForm(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid") - - type: str - nickname: str | None = None - config: dict = pydantic.Field(default_factory=dict) - storage: StorageID | None = None - - -class SourceUpdateForm(pydantic.BaseModel): - """Only submitted fields are written; config is replaced, not deep-merged.""" - - model_config = pydantic.ConfigDict(extra="forbid") - - nickname: str | None = None - config: dict = pydantic.Field(default_factory=dict) - storage: StorageID | None = None - - -class CollectAt(sqlmodel.SQLModel): - """Historical migration-only value retained for the append-only baseline.""" - - day_of_week: int | None = None - hour: int = 0 - minute: int = 0 - - -class CollectAtType(sqlmodel.TypeDecorator): - """Historical migration-only PostgreSQL type; not a current Source field.""" - - impl = sqlalchemy.dialects.postgresql.JSONB - cache_ok = True - - def process_bind_param(self, value, dialect): - del dialect - return value.model_dump() if isinstance(value, CollectAt) else value - - def process_result_value(self, value, dialect): - del dialect - return CollectAt.model_validate(value) if value else None - - -class SourceCollectParameters(pydantic.BaseModel): - """Parameters of the exact ordinary Source Job type.""" - - model_config = pydantic.ConfigDict(extra="forbid") - - source: SourceID - config: dict[str, typing.Any] = pydantic.Field(default_factory=dict) - - -class SourceBackfillParameters(SourceCollectParameters): - """Parameters of the exact historical Source Job type.""" - - -class SourceTypesModel(sqlmodel.SQLModel, table=True): - __tablename__: str = "sources_types" # type: ignore - - id: str = sqlmodel.Field(sa_column=sqlalchemy.Column(sqlalchemy.Text, primary_key=True)) - """Type of source. - - An absolute import path to the module where souce class at. - """ - description: str = sqlmodel.Field(sa_column=sqlalchemy.Column(sqlalchemy.Text)) - """Description of this source type.""" - config_schema: dict = sqlmodel.Field( - sa_column=sqlalchemy.Column( - sqlalchemy.dialects.postgresql.JSONB, - server_default=sqlalchemy.text("'{}'::jsonb"), - nullable=False, - ), - default=dict, - ) - collect_config_schema: dict = sqlmodel.Field( - sa_column=sqlalchemy.Column( - sqlalchemy.dialects.postgresql.JSONB, - server_default=sqlalchemy.text("'{}'::jsonb"), - nullable=False, - ), - default=dict, - ) - backfill_config_schema: dict | None = sqlmodel.Field( - sa_column=sqlalchemy.Column( - sqlalchemy.dialects.postgresql.JSONB, - nullable=True, - ), - default=None, - ) - - -class SourceModel(sqlmodel.SQLModel, table=True): - __tablename__: str = "sources" # type: ignore - - id: Opt[SourceID] = sqlmodel.Field( - sa_column=sqlalchemy.Column(sqlalchemy.Integer, primary_key=True, autoincrement=True), - default=None, - ) - type: str = sqlmodel.Field( - sa_column=sqlalchemy.Column( - sqlalchemy.Text, - sqlalchemy.ForeignKey("sources_types.id", onupdate="CASCADE", ondelete="CASCADE"), - ) - ) - """Type of source. - - An absolute import path to the module where souce class at. - When delete a source type, all sources of this type will be deleted too. - """ - nickname: Opt[str] = sqlmodel.Field( - default=None, - sa_column=sqlalchemy.Column(sqlalchemy.Text, nullable=True), - ) - config: dict = sqlmodel.Field( - sa_column=sqlalchemy.Column( - sqlalchemy.dialects.postgresql.JSONB, - server_default=sqlalchemy.text("'{}'::jsonb"), - nullable=False, - ), - default=dict, - ) - storage: Opt[StorageID] = sqlmodel.Field( - default=None, - sa_column=sqlalchemy.Column( - sqlalchemy.Integer, - sqlalchemy.ForeignKey("storages.id", onupdate="CASCADE", ondelete="RESTRICT"), - nullable=True, - ), - ) - block: Opt[BlockID] = sqlmodel.Field( - default=None, - sa_column=sqlalchemy.Column( - sqlalchemy.Integer, - sqlalchemy.ForeignKey("blocks.id", onupdate="CASCADE", ondelete="SET NULL"), - nullable=True, - unique=True, - ), - ) - state: dict = sqlmodel.Field( - sa_column=sqlalchemy.Column( - sqlalchemy.dialects.postgresql.JSONB, - server_default=sqlalchemy.text("'{}'::jsonb"), - nullable=False, - ), - default_factory=dict, - ) - """Store source-specific state (e.g., last_uid for mail, - latest_tweet_id for twitter).""" - created_at: datetime.datetime = sqlmodel.Field( - default=None, - sa_column=sqlalchemy.Column( - sqlalchemy.TIMESTAMP(timezone=True), - nullable=False, - server_default=sqlalchemy.text("CURRENT_TIMESTAMP"), - ), - ) - updated_at: datetime.datetime = sqlmodel.Field( - default=None, - sa_column=sqlalchemy.Column( - sqlalchemy.TIMESTAMP(timezone=True), - nullable=False, - server_default=sqlalchemy.text("CURRENT_TIMESTAMP"), - ), - ) +if TYPE_CHECKING: + from inkcre_core.schemas.source.main import * + from inkcre_core.schemas.source.main import CollectAtType as CollectAtType diff --git a/app/settings.py b/app/settings.py index 88d3ff2a..fc07a862 100644 --- a/app/settings.py +++ b/app/settings.py @@ -9,8 +9,8 @@ from pydantic import Field, field_validator from pydantic_settings import BaseSettings, SettingsConfigDict -from app.database_contract.constants import JWT_MINIMUM_SECRET_BYTES -from libs.obsrv.setting import ObsrvSetting +from inkcre_core.database_contract.constants import JWT_MINIMUM_SECRET_BYTES +from inkcre_core.libs.obsrv.setting import ObsrvSetting class Settings(BaseSettings): diff --git a/app/validation.py b/app/validation.py index beaee8e2..db4ed96e 100644 --- a/app/validation.py +++ b/app/validation.py @@ -1,31 +1,5 @@ -"""Path projection for nested input contracts; validation stays with their owners.""" +"""Legacy import; use inkcre_core for new consumers.""" -import contextlib -import typing +from inkcre_core.validation import input_path -import jsonschema # pyrefly: ignore[untyped-import] -import pydantic - - -@contextlib.contextmanager -def input_path(*prefix: str | int): - """Attach a form nesting path to native errors without validating again.""" - try: - yield - except pydantic.ValidationError as error: - details = [ - {**item, "loc": (*prefix, *item["loc"])} for item in error.errors(include_url=False) - ] - raise pydantic.ValidationError.from_exception_data( - error.title, typing.cast(typing.Any, details) - ) from error - except jsonschema.ValidationError as error: - raise jsonschema.ValidationError( - error.message, - validator=error.validator, - validator_value=error.validator_value, - instance=error.instance, - schema=error.schema, - path=(*prefix, *error.absolute_path), - schema_path=error.absolute_schema_path, - ) from error +__all__ = ["input_path"] diff --git a/docs/30-unit-tdd/business-pipeline-and-authority.md b/docs/30-unit-tdd/business-pipeline-and-authority.md index caab6cf9..ea1b7778 100644 --- a/docs/30-unit-tdd/business-pipeline-and-authority.md +++ b/docs/30-unit-tdd/business-pipeline-and-authority.md @@ -17,9 +17,9 @@ implementation direction; it must not redefine Peer wire behavior or shared capa 在以下情况先读这里： - 修改 `app/business/extension/` -- 修改 `app/business/source/` -- 修改 `app/business/info_base/` -- 修改 `app/business/ai/` 或 application/use capability +- 修改 `inkcre_core/business/source/` +- 修改 `inkcre_core/business/info_base/` +- 修改 `inkcre_core/business/ai/` 或 application/use capability - 修改这些子树之间的 ownership / dependency / authority ## Structure @@ -59,7 +59,7 @@ implementation direction; it must not redefine Peer wire behavior or shared capa ### 数据库事务边界 -`app/business` 拥有应用用例、业务规则及事务组合；`app/persistence` 按业务责任分组，拥有 +`inkcre_core/business` 拥有应用用例、业务规则及事务组合；`inkcre_core/persistence` 按业务责任分组，拥有 session-bound repository 和 UoW 的数据库工厂实现。用例决定何时进入、退出作用域，UoW 工厂负责 实际创建 session 和原生事务 framing。repository 不反向依赖 business。 HTTP route 只解析请求并调用 business 入口，不直接查询 repository；例如 `get_entity_records` @@ -83,12 +83,12 @@ AI 调用前释放读取 scope，返回后重新检查已有派生，再在一 配置 HTTP 使用 DeploymentConfigService，AI、Agent 定义和 Peer 的数据库入口各自使用所属 UoW。 AI provider 调用和 Peer outbound 执行前结束数据库作用域；Peer 的内存 capability 注册仍是同步函数。 -`app/engine.py` 是异步 factory 的 authority，lifespan 在业务运行资源退出后 dispose 异步 engine。 +`inkcre_core.Core` 每实例拥有异步 factory；`app/core.py` 只负责服务装配与 ASGI 绑定。lifespan 在业务运行资源退出后 await Core.aclose()，外部借用的 engine 不由 Core dispose。 UoW 不跨并发任务共享。应用操作不接受可选 session；共同原子提交必须组合事务内操作，而不调用 独立提交入口。`expire_on_commit=False` 允许已提交的已加载字段在 scope 外读取，不授权隐式延迟查询。 ExtensionStateService 与 Host 的数据库操作已异步化，state/config transform 在行锁内同步执行， -SQL 位于 app/persistence/extension；Source catalog 批量同步提供 SDK async startup 所需能力。 +SQL 位于 inkcre_core/persistence/extension；Source catalog 批量同步提供 SDK async startup 所需能力。 启用／停用的持久化失败补偿和启动取消清理属于 Host；Peer 广播失败不撤销已提交的 disabled 状态。 Registry origin 查询结束后才执行网络／wheel 获取。旧同步 Host 合同通过 Host 0.2 窗口隔离。 diff --git a/docs/30-unit-tdd/lexical-retrieval.md b/docs/30-unit-tdd/lexical-retrieval.md index 810a2b95..11e92353 100644 --- a/docs/30-unit-tdd/lexical-retrieval.md +++ b/docs/30-unit-tdd/lexical-retrieval.md @@ -109,5 +109,5 @@ contains corpus aliases, expected phrases or test-shaped IDs. - recursive graph indexing - implicit maintenance during a retrieval request -数据库查询与 projection upsert 由 `app/persistence/lexical_retrieval/` 执行。业务入口原生 await， +数据库查询与 projection upsert 由 `inkcre_core/persistence/lexical_retrieval/` 执行。业务入口原生 await， 每个维护 batch 在短事务中一次 upsert 全部有效投影；Resolver 投影不占用该写事务。 diff --git a/docs/30-unit-tdd/semantic-retrieval.md b/docs/30-unit-tdd/semantic-retrieval.md index cb3e4f18..7d78a274 100644 --- a/docs/30-unit-tdd/semantic-retrieval.md +++ b/docs/30-unit-tdd/semantic-retrieval.md @@ -101,6 +101,6 @@ vectors, and generated graph rows are never committed as authority. - generic capability invocation endpoints or delegation jobs - persistent Agent Thread/checkpoint infrastructure -数据库查询与 embedding upsert 由 `app/persistence/semantic_retrieval/` 执行。每个 provider batch +数据库查询与 embedding upsert 由 `inkcre_core/persistence/semantic_retrieval/` 执行。每个 provider batch 先完成向量数量与维度校验，再在同一短事务中分别批量写入 Block 和 Relation embeddings； 任一种写入失败都会回滚整个 batch。profile 与检索读取同样原生 await，不使用同步 session。 diff --git a/docs/40-deployment/development-environment.md b/docs/40-deployment/development-environment.md index 42dcbd34..34521c8d 100644 --- a/docs/40-deployment/development-environment.md +++ b/docs/40-deployment/development-environment.md @@ -93,7 +93,7 @@ For Python-only iteration against an already initialized database: ```bash cp .env.example .env -pdm install -G dev --frozen-lockfile +pdm install -G app -G dev --frozen-lockfile pdm run doctor pdm run check pdm run dev diff --git a/docs/_shared b/docs/_shared index 4e323b49..b3334622 160000 --- a/docs/_shared +++ b/docs/_shared @@ -1 +1 @@ -Subproject commit 4e323b49ffcd00011f7e008fbb53866c6bec5a27 +Subproject commit b33346225b669a6ac93fcac12a135178a46533c8 diff --git a/inkcre_core/AGENTS.md b/inkcre_core/AGENTS.md new file mode 100644 index 00000000..0c4b865b --- /dev/null +++ b/inkcre_core/AGENTS.md @@ -0,0 +1,7 @@ +# Independent Python SDK + +- Ordinary SDK imports and operations must not depend on `app`, FastAPI, Starlette, the service scheduler, environment loading, or an application singleton. +- `Core` owns its engine factory and task-local binding. Each operation still creates its own UoW; never put a live session in the context or share it across concurrent tasks. +- Keep shared protocol IDs, model fields and transaction guarantees stable when moving modules. Legacy service aliases must resolve to the same model/service objects. +- Consumer entrypoints and ownership are documented in the repository README. Do not expose persistence internals as a stable public facade merely because they are packaged. +- Check the built wheel with `pdm run check:core-package`; it is consumed outside this checkout with application imports rejected. This check does not contact a database. diff --git a/inkcre_core/__init__.py b/inkcre_core/__init__.py new file mode 100644 index 00000000..cd38d654 --- /dev/null +++ b/inkcre_core/__init__.py @@ -0,0 +1,16 @@ +"""InKCre SDK. Importing contracts never initializes application resources.""" + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from .context import Core, CoreOptions, current_core + +__all__ = ["Core", "CoreOptions", "current_core"] + + +def __getattr__(name: str): + if name not in __all__: + raise AttributeError(name) + from . import context + + return getattr(context, name) diff --git a/inkcre_core/auth.py b/inkcre_core/auth.py new file mode 100644 index 00000000..8b6a9734 --- /dev/null +++ b/inkcre_core/auth.py @@ -0,0 +1,78 @@ +"""Peer JWT encoding and validation without application configuration.""" + +import math +import time +import jwt +from .database_contract.constants import JWT_ALGORITHM, JWT_AUDIENCE, JWT_ISSUER, JWT_ROLE + + +def decode_peer_jwt( + token: str, + secret: str, + *, + now: float | None = None, +) -> dict: + """Validate the canonical peer JWT contract used by every HTTP surface.""" + claims = jwt.decode( + token, + secret, + algorithms=[JWT_ALGORITHM], + audience=JWT_AUDIENCE, + issuer=JWT_ISSUER, + options={ + "require": ["role", "iss", "aud", "iat", "exp"], + "verify_exp": False, + "verify_iat": False, + }, + ) + issued_at = claims["iat"] + expires_at = claims["exp"] + valid_numeric_dates = ( + isinstance(issued_at, (int, float)) + and not isinstance(issued_at, bool) + and isinstance(expires_at, (int, float)) + and not isinstance(expires_at, bool) + ) + if not valid_numeric_dates: + raise jwt.exceptions.InvalidTokenError("JWT claim contract mismatch") + try: + issued_at_number = float(issued_at) + expires_at_number = float(expires_at) + except OverflowError as error: + raise jwt.exceptions.InvalidTokenError("JWT claim contract mismatch") from error + current_epoch = time.time() if now is None else now + if ( + claims["role"] != JWT_ROLE + or not math.isfinite(issued_at_number) + or not math.isfinite(expires_at_number) + or issued_at_number > current_epoch + 60 + or expires_at_number <= current_epoch + or expires_at_number <= issued_at_number + ): + raise jwt.exceptions.InvalidTokenError("JWT claim contract mismatch") + return claims + + +def create_peer_jwt( + secret: str, + *, + now: float | None = None, + lifetime_seconds: int = 60, +) -> str: + """Sign a Peer JWT; the default is client policy, not an admission limit.""" + if not secret: + raise ValueError("Peer JWT secret is required") + if lifetime_seconds <= 0: + raise ValueError("Peer JWT lifetime must be positive") + issued_at = int(time.time() if now is None else now) + return jwt.encode( + { + "role": JWT_ROLE, + "iss": JWT_ISSUER, + "aud": JWT_AUDIENCE, + "iat": issued_at, + "exp": issued_at + lifetime_seconds, + }, + secret, + algorithm=JWT_ALGORITHM, + ) diff --git a/inkcre_core/business/AGENTS.md b/inkcre_core/business/AGENTS.md new file mode 100644 index 00000000..3bbce370 --- /dev/null +++ b/inkcre_core/business/AGENTS.md @@ -0,0 +1,12 @@ +# Business Units + +> Applies to `inkcre_core/business/`; deeper guides own local hazards. + +- Read `docs/30-unit-tdd/business-pipeline-and-authority.md` before a change crosses business subtrees. +- `info_base` owns graph persistence; sources and extensions propose commands but do not copy generic persistence. +- AI execution remains graph-blind. Agent and organization layers may compose it but must not move graph or resolver policy into AI. +- Semantic and lexical retrieval own separate derived records and ranking policy; neither owns graph facts or mutates the graph while reading. +- Global Job/Cron machinery owns durable execution occurrence and claim lifecycle; domain units retain typed payload and effect semantics. +- Peer owns identity, capability, lease, and delegation transport facts; typed business codecs remain with their business owner. +- Application use cases choose UoW scopes; raw sessions stay in persistence. Composed operations require a UoW and never end its transaction. Each concurrent task owns a separate UoW; external I/O runs outside DB scopes. +- Required check: run tests for every affected business owner and the repository import-boundary checks. diff --git a/inkcre_core/business/__init__.py b/inkcre_core/business/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/business/agent/AGENTS.md b/inkcre_core/business/agent/AGENTS.md similarity index 100% rename from app/business/agent/AGENTS.md rename to inkcre_core/business/agent/AGENTS.md diff --git a/inkcre_core/business/agent/__init__.py b/inkcre_core/business/agent/__init__.py new file mode 100644 index 00000000..33fcb1d3 --- /dev/null +++ b/inkcre_core/business/agent/__init__.py @@ -0,0 +1,40 @@ +"""Reusable Agent definitions, typed Tools and in-memory Thread execution.""" + +from .contracts import ( + AgentError, + AgentNotFoundError, + AgentToolBindingError, + AgentTurnActiveError, + BoundAgentTool, + DuplicateAgentToolRegistrationError, + MissingAgentToolError, + ToolExecutionError, +) +from .main import AgentManager +from .bootstrap import register_core_agent_tools +from .persistence import ( + InMemoryThreadPersistenceBackend, + ThreadID, + ThreadPersistenceBackend, + ThreadState, +) +from .thread import Thread, TurnTermination + +__all__ = [ + "AgentError", + "AgentManager", + "register_core_agent_tools", + "AgentNotFoundError", + "AgentToolBindingError", + "AgentTurnActiveError", + "BoundAgentTool", + "DuplicateAgentToolRegistrationError", + "InMemoryThreadPersistenceBackend", + "MissingAgentToolError", + "Thread", + "ThreadID", + "ThreadPersistenceBackend", + "ThreadState", + "ToolExecutionError", + "TurnTermination", +] diff --git a/inkcre_core/business/agent/bootstrap.py b/inkcre_core/business/agent/bootstrap.py new file mode 100644 index 00000000..0f69e4e4 --- /dev/null +++ b/inkcre_core/business/agent/bootstrap.py @@ -0,0 +1,16 @@ +"""Explicit registration entrypoint for core-owned Agent Tool controllers.""" + + +def register_core_agent_tools() -> None: + """Import every core Tool owner, independent of HTTP Sink delivery. + + Decorators register Tools when their modules load; Python's import cache makes + repeated bootstrap calls harmless. Entry points must call this explicitly, + rather than rely on routes or package re-exports importing the owners first. + """ + from inkcre_core.business.graph_navigation_retrieval import tools as _graph_tools + from inkcre_core.business.info_base import tools as _info_base_tools + from inkcre_core.business.info_base.resolver import tools as _resolver_tools + from inkcre_core.business.organization import tools as _organization_tools + + del _graph_tools, _info_base_tools, _resolver_tools, _organization_tools diff --git a/inkcre_core/business/agent/contracts.py b/inkcre_core/business/agent/contracts.py new file mode 100644 index 00000000..5e5e05e5 --- /dev/null +++ b/inkcre_core/business/agent/contracts.py @@ -0,0 +1,59 @@ +"""Agent Tool binding and runtime failure contracts.""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable +from dataclasses import dataclass +import typing + +import pydantic + +from inkcre_core.schemas.ai import FunctionTool, JSONValue + + +AgentToolHandler: typing.TypeAlias = Callable[ + [pydantic.BaseModel], + JSONValue | Awaitable[JSONValue], +] +AgentToolInputModelFactory: typing.TypeAlias = Callable[[], type[pydantic.BaseModel]] + + +class AgentError(RuntimeError): + """Base failure exposed by the Agent domain.""" + + +class AgentNotFoundError(AgentError): + pass + + +class AgentToolBindingError(AgentError): + pass + + +class DuplicateAgentToolRegistrationError(AgentToolBindingError): + pass + + +class MissingAgentToolError(AgentToolBindingError): + pass + + +class AgentTurnActiveError(AgentError): + pass + + +class ToolExecutionError(Exception): + """A Tool-owned actionable failure intended for the model.""" + + def __init__(self, content: JSONValue): + super().__init__("Agent Tool execution failed") + self.content = content + + +@dataclass(frozen=True) +class BoundAgentTool: + """One run-local Tool schema, validation model and code-owned handler.""" + + definition: FunctionTool + input_model: type[pydantic.BaseModel] + handler: AgentToolHandler diff --git a/inkcre_core/business/agent/debug.py b/inkcre_core/business/agent/debug.py new file mode 100644 index 00000000..9d17001d --- /dev/null +++ b/inkcre_core/business/agent/debug.py @@ -0,0 +1,51 @@ +"""Opt-in development traces through the existing logging backend.""" + +import asyncio +import datetime +import json +import sys +import typing +import uuid + +import pydantic + +from inkcre_core.context import current_core +from inkcre_core.libs.obsrv.log_record import ENABLE_LOG_BACKEND, TRACE_ID +from inkcre_core.libs.obsrv.main import get_logger + + +def _json_value(value: typing.Any) -> typing.Any: + if isinstance(value, pydantic.BaseModel): + return value.model_dump(mode="python") + if isinstance(value, bytes): + return {"omitted_binary_bytes": len(value)} + if isinstance(value, datetime.datetime | datetime.date): + return value.isoformat() + if isinstance(value, uuid.UUID): + return str(value) + raise TypeError(f"Unsupported Agent trace value: {type(value).__name__}") + + +def _emit(event: str, thread_id: str, payload: dict[str, typing.Any]) -> None: + token = ENABLE_LOG_BACKEND.set(True) + try: + envelope = { + "event": event, + "thread_id": thread_id, + "trace_id": TRACE_ID.get(), + **payload, + } + get_logger().getChild("agent.debug").info( + json.dumps(envelope, ensure_ascii=False, default=_json_value), + extra={"event": event, "agent_thread_id": thread_id}, + ) + except Exception as error: + # A debug destination must never replace the Agent's real outcome. + sys.stderr.write(f"Agent debug trace unavailable: {type(error).__name__}\n") + finally: + ENABLE_LOG_BACKEND.reset(token) + + +async def trace(event: str, thread_id: uuid.UUID, **payload: typing.Any) -> None: + if current_core().options.agent_debug: + await asyncio.to_thread(_emit, event, str(thread_id), payload) diff --git a/inkcre_core/business/agent/main.py b/inkcre_core/business/agent/main.py new file mode 100644 index 00000000..5a6a776c --- /dev/null +++ b/inkcre_core/business/agent/main.py @@ -0,0 +1,244 @@ +"""Persisted Agent definition loading and exact Tool registry binding.""" + +from __future__ import annotations + +from dataclasses import dataclass +import inspect +import typing + +import pydantic + +from inkcre_core.business.ai import AIExecutionRequirement, AIManager +from inkcre_core.schemas import AgentDefinitionModel +from inkcre_core.schemas.agent import AgentID, AgentForm, AgentUpdateForm +from inkcre_core.schemas.ai import FunctionTool, SystemMessage, UserMessage + +from inkcre_core.persistence.agent.uow import agent_uow + +from .contracts import ( + AgentNotFoundError, + AgentToolBindingError, + AgentToolHandler, + AgentToolInputModelFactory, + BoundAgentTool, + DuplicateAgentToolRegistrationError, + MissingAgentToolError, +) +from .persistence import ( + InMemoryThreadPersistenceBackend, + ThreadPersistenceBackend, + ThreadState, +) +from .thread import Thread +from .debug import trace + + +HandlerT = typing.TypeVar("HandlerT", bound=typing.Callable[..., typing.Any]) + + +@dataclass(frozen=True) +class _ToolRegistration: + handler: AgentToolHandler + original_handler: typing.Callable[..., typing.Any] + description: str + input_model: type[pydantic.BaseModel] + input_model_factory: AgentToolInputModelFactory | None + + def bind(self, tool_id: str) -> BoundAgentTool: + input_model = ( + self.input_model_factory() + if self.input_model_factory is not None + else self.input_model + ) + if not isinstance(input_model, type) or not issubclass(input_model, pydantic.BaseModel): + raise AgentToolBindingError( + f"Agent Tool {tool_id!r} did not produce a Pydantic input model" + ) + return BoundAgentTool( + definition=FunctionTool( + id=tool_id, + description=self.description, + input_schema=input_model.model_json_schema(), + ), + input_model=input_model, + handler=self.handler, + ) + + +class AgentManager: + """Agent-definition execution and peer-local exact Tool registry.""" + + _TOOLS: dict[str, _ToolRegistration] = {} + _persistence: ThreadPersistenceBackend = InMemoryThreadPersistenceBackend() + + @classmethod + async def get_definition(cls, agent_id: AgentID) -> AgentDefinitionModel | None: + async with agent_uow() as agents: + return await agents.get(agent_id) + + @classmethod + async def list_definitions( + cls, *, limit: int | None = None, cursor: int | None = None + ) -> tuple[list[AgentDefinitionModel], int | None]: + async with agent_uow() as agents: + return await agents.list(limit=limit, cursor=cursor) + + @classmethod + async def create_definition(cls, form: AgentForm) -> AgentDefinitionModel: + async with agent_uow() as agents: + record = await agents.create(form) + return record + + @classmethod + async def update_definition( + cls, agent_id: AgentID, form: AgentUpdateForm + ) -> AgentDefinitionModel: + async with agent_uow() as agents: + record = await agents.get(agent_id, for_update=True) + if record is None: + raise AgentNotFoundError(f"Agent {agent_id} does not exist") + changes = form.model_dump(exclude_unset=True) + candidate = AgentForm.model_validate( + {**{field: getattr(record, field) for field in AgentForm.model_fields}, **changes} + ) + for field in changes: + setattr(record, field, getattr(candidate, field)) + await agents.save(record) + return record + + @classmethod + async def delete_definition(cls, agent_id: AgentID) -> bool: + async with agent_uow() as agents: + record = await agents.get(agent_id) + if record is None: + return False + await agents.delete(record) + return True + + @classmethod + def list_tools( + cls, *, limit: int | None = None, cursor: str | None = None + ) -> tuple[list[dict[str, str]], str | None]: + ids = sorted(key for key in cls._TOOLS if cursor is None or key > cursor) + more = limit is not None and len(ids) > limit + ids = ids[:limit] + return [{"id": key, "description": cls._TOOLS[key].description} for key in ids], ids[ + -1 + ] if more else None + + @classmethod + def get_tool(cls, tool_id: str) -> FunctionTool: + registration = cls._TOOLS.get(tool_id) + if registration is None: + raise MissingAgentToolError(f"Agent Tool {tool_id!r} is not registered") + return registration.bind(tool_id).definition + + @classmethod + async def can_execute(cls, agent_id: AgentID, input_modality: str) -> bool: + """Return static local eligibility for one Agent and canonical input modality.""" + definition = await cls.get_definition(agent_id) + if definition is None: + return False + try: + cls._bind_tools(definition.tools) + except MissingAgentToolError: + return False + requires_tools = bool(definition.tools) or definition.tool_choice is not None + return await AIManager.can_execute( + definition.model, + AIExecutionRequirement( + capability="chat", + input_modalities=frozenset({"text", input_modality}), + output_modalities=frozenset({"text"}), + features=frozenset({"tool_calling"}) if requires_tools else frozenset(), + tool_choice=definition.tool_choice, + ), + ) + + @classmethod + def tool( + cls, + tool_id: str, + *, + description: str, + input_model_factory: AgentToolInputModelFactory | None = None, + ) -> typing.Callable[[HandlerT], HandlerT]: + """Register one function as a typed Agent Tool and return it unchanged.""" + if not tool_id: + raise AgentToolBindingError("Agent Tool ID must not be empty") + + def decorator(handler: HandlerT) -> HandlerT: + signature = inspect.signature(handler) + parameters = tuple(signature.parameters.values()) + if len(parameters) != 1: + raise AgentToolBindingError( + f"Agent Tool {tool_id!r} handler must accept exactly one input model" + ) + hints = typing.get_type_hints(handler) + annotation = hints.get(parameters[0].name) + if not isinstance(annotation, type) or not issubclass(annotation, pydantic.BaseModel): + raise AgentToolBindingError( + f"Agent Tool {tool_id!r} input must be annotated with a Pydantic model" + ) + + existing = cls._TOOLS.get(tool_id) + if existing is not None: + if ( + existing.original_handler is handler + and existing.description == description + and existing.input_model_factory is input_model_factory + ): + return handler + raise DuplicateAgentToolRegistrationError( + f"Agent Tool {tool_id!r} is already registered" + ) + + cls._TOOLS[tool_id] = _ToolRegistration( + handler=typing.cast(AgentToolHandler, handler), + original_handler=handler, + description=description, + input_model=annotation, + input_model_factory=input_model_factory, + ) + return handler + + return decorator + + @classmethod + async def run(cls, agent_id: AgentID, initial_message: UserMessage) -> Thread: + """Create one active Thread from a persisted Agent definition snapshot.""" + definition = await cls.get_definition(agent_id) + if definition is None: + raise AgentNotFoundError(f"Agent {agent_id} does not exist") + + bound_tools = cls._bind_tools(definition.tools) + state = ThreadState( + model=definition.model, + tools=tuple(tool.definition for tool in bound_tools), + tool_choice=definition.tool_choice, + max_model_calls_per_turn=definition.max_model_calls_per_turn, + messages=(SystemMessage(content=definition.system_prompt),), + ) + thread_id, persisted = await cls._persistence.create(state) + await trace( + "agent.thread.created", + thread_id, + agent_id=agent_id, + agent_name=definition.name, + state=persisted, + ) + thread = Thread(thread_id, persisted, cls._persistence, bound_tools) + thread.start_turn(initial_message) + return thread + + @classmethod + def _bind_tools(cls, tool_ids: tuple[str, ...]) -> tuple[BoundAgentTool, ...]: + bound: list[BoundAgentTool] = [] + for tool_id in tool_ids: + registration = cls._TOOLS.get(tool_id) + if registration is None: + raise MissingAgentToolError( + f"Agent definition references unavailable Tool {tool_id!r}" + ) + bound.append(registration.bind(tool_id)) + return tuple(bound) diff --git a/inkcre_core/business/agent/persistence.py b/inkcre_core/business/agent/persistence.py new file mode 100644 index 00000000..46565f92 --- /dev/null +++ b/inkcre_core/business/agent/persistence.py @@ -0,0 +1,147 @@ +"""Replaceable whole-Thread persistence with an in-memory MVP backend.""" + +from __future__ import annotations + +import asyncio +from abc import ABC, abstractmethod +import typing +import uuid + +import pydantic + +from inkcre_core.schemas.ai import ( + AIModelID, + AssistantMessage, + FunctionTool, + Message, + ToolChoice, + validate_message_history, +) + + +ThreadID: typing.TypeAlias = uuid.UUID + + +def validate_thread_message_history(value: typing.Any) -> tuple[Message, ...]: + """Allow strict history plus one recoverable trailing ToolCall message.""" + messages = tuple(value) + if messages and isinstance(messages[-1], AssistantMessage) and messages[-1].tool_calls: + validate_message_history(messages[:-1]) + return messages + return validate_message_history(messages) + + +class ThreadState(pydantic.BaseModel): + """Serializable Thread snapshot owned by a persistence backend.""" + + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + model: AIModelID + tools: tuple[FunctionTool, ...] + tool_choice: ToolChoice | None + max_model_calls_per_turn: int = pydantic.Field(gt=0) + messages: tuple[Message, ...] + + @pydantic.field_validator("tools") + @classmethod + def unique_tools(cls, value: tuple[FunctionTool, ...]) -> tuple[FunctionTool, ...]: + tool_ids = tuple(tool.id for tool in value) + if len(tool_ids) != len(set(tool_ids)): + raise ValueError("Thread Tool IDs must be unique") + return value + + @pydantic.field_validator("messages") + @classmethod + def closed_or_recoverable_history(cls, value: tuple[Message, ...]) -> tuple[Message, ...]: + return validate_thread_message_history(value) + + +class ThreadPersistenceBackend(ABC): + """Atomic persistence boundary for complete Thread snapshots.""" + + @abstractmethod + async def create(self, state: ThreadState) -> tuple[ThreadID, ThreadState]: + raise NotImplementedError + + @abstractmethod + async def read(self, thread_id: ThreadID) -> ThreadState: + raise NotImplementedError + + @abstractmethod + async def append( + self, + thread_id: ThreadID, + messages: tuple[Message, ...], + ) -> ThreadState: + raise NotImplementedError + + @abstractmethod + async def discard_trailing_incomplete_tool_calls( + self, + thread_id: ThreadID, + ) -> ThreadState: + raise NotImplementedError + + +class InMemoryThreadPersistenceBackend(ThreadPersistenceBackend): + """Process-local backend preserving atomic Thread message updates.""" + + def __init__(self) -> None: + self._states: dict[ThreadID, ThreadState] = {} + self._locks: dict[ThreadID, asyncio.Lock] = {} + + async def create(self, state: ThreadState) -> tuple[ThreadID, ThreadState]: + thread_id = uuid.uuid4() + self._states[thread_id] = state + self._locks[thread_id] = asyncio.Lock() + return thread_id, state + + async def read(self, thread_id: ThreadID) -> ThreadState: + try: + return self._states[thread_id] + except KeyError as error: + raise KeyError(f"Thread {thread_id} does not exist") from error + + async def append( + self, + thread_id: ThreadID, + messages: tuple[Message, ...], + ) -> ThreadState: + async with self._lock(thread_id): + current = await self.read(thread_id) + updated = ThreadState( + model=current.model, + tools=current.tools, + tool_choice=current.tool_choice, + max_model_calls_per_turn=current.max_model_calls_per_turn, + messages=current.messages + messages, + ) + self._states[thread_id] = updated + return updated + + async def discard_trailing_incomplete_tool_calls( + self, + thread_id: ThreadID, + ) -> ThreadState: + async with self._lock(thread_id): + current = await self.read(thread_id) + if not current.messages: + return current + trailing = current.messages[-1] + if not isinstance(trailing, AssistantMessage) or not trailing.tool_calls: + return current + updated = ThreadState( + model=current.model, + tools=current.tools, + tool_choice=current.tool_choice, + max_model_calls_per_turn=current.max_model_calls_per_turn, + messages=current.messages[:-1], + ) + self._states[thread_id] = updated + return updated + + def _lock(self, thread_id: ThreadID) -> asyncio.Lock: + try: + return self._locks[thread_id] + except KeyError as error: + raise KeyError(f"Thread {thread_id} does not exist") from error diff --git a/inkcre_core/business/agent/projection.py b/inkcre_core/business/agent/projection.py new file mode 100644 index 00000000..c1ea3c49 --- /dev/null +++ b/inkcre_core/business/agent/projection.py @@ -0,0 +1,45 @@ +"""JSON projection shared by domain-owned Agent Tool controllers.""" + +import dataclasses +import typing + +import pydantic + +from inkcre_core.schemas.ai import JSONValue + + +_JSON_ADAPTER = pydantic.TypeAdapter(JSONValue) + + +def project_json(value: typing.Any) -> JSONValue: + """Project ordinary domain values without admitting binary Tool results.""" + if _contains_bytes(value): + raise TypeError("Binary values are unavailable through JSON Agent Tools") + if isinstance(value, pydantic.BaseModel): + projected = value.model_dump(mode="json") + elif dataclasses.is_dataclass(value) and not isinstance(value, type): + projected = dataclasses.asdict(value) + else: + projected = pydantic.TypeAdapter(typing.Any).dump_python( + typing.cast(typing.Any, value), + mode="json", + ) + return _JSON_ADAPTER.validate_python(projected) + + +def _contains_bytes(value: typing.Any) -> bool: + if isinstance(value, bytes): + return True + if isinstance(value, pydantic.BaseModel): + return any( + _contains_bytes(getattr(value, field)) for field in value.__class__.model_fields + ) + if dataclasses.is_dataclass(value) and not isinstance(value, type): + return any( + _contains_bytes(getattr(value, field.name)) for field in dataclasses.fields(value) + ) + if isinstance(value, dict): + return any(_contains_bytes(item) for item in value.values()) + if isinstance(value, list | tuple | set | frozenset): + return any(_contains_bytes(item) for item in value) + return False diff --git a/inkcre_core/business/agent/thread.py b/inkcre_core/business/agent/thread.py new file mode 100644 index 00000000..a6d2aa13 --- /dev/null +++ b/inkcre_core/business/agent/thread.py @@ -0,0 +1,316 @@ +"""Cancellable Agent Thread and per-turn model/Tool orchestration.""" + +from __future__ import annotations + +import asyncio +from enum import StrEnum +import inspect +import hashlib +import json +import time +import traceback +import typing + +import pydantic + +from inkcre_core.business.ai import AIManager +from inkcre_core.schemas.ai import ( + AssistantMessage, + JSONValue, + ToolCall, + ToolResult, + ToolResultMessage, + UserMessage, +) + +from .contracts import AgentTurnActiveError, BoundAgentTool, ToolExecutionError +from .persistence import ThreadID, ThreadPersistenceBackend, ThreadState +from .debug import trace +from inkcre_core.libs.obsrv.main import get_logger +from inkcre_core.libs.obsrv.telemetry import operation + + +logger = get_logger().getChild("agent.thread") + + +class TurnTermination(StrEnum): + COMPLETED = "completed" + MAX_MODEL_CALLS = "max_model_calls" + + +class Thread: + """One persisted Agent-definition snapshot with at most one active Turn.""" + + def __init__( + self, + thread_id: ThreadID, + state: ThreadState, + persistence: ThreadPersistenceBackend, + tools: tuple[BoundAgentTool, ...], + ) -> None: + self.id = thread_id + self._state = state + self._persistence = persistence + self._tools = {tool.definition.id: tool for tool in tools} + self.current_turn: asyncio.Task[TurnTermination] | None = None + self._turn_index = 0 + self._model_calls = 0 + + @property + def messages(self): + return self._state.messages + + @property + def model(self): + return self._state.model + + @property + def tools(self): + return self._state.tools + + @property + def tool_choice(self): + return self._state.tool_choice + + @property + def max_model_calls_per_turn(self): + return self._state.max_model_calls_per_turn + + async def refresh(self) -> ThreadState: + """Refresh the local view from the persistence authority.""" + self._state = await self._persistence.read(self.id) + return self._state + + def start_turn(self, input: UserMessage) -> asyncio.Task[TurnTermination]: + """Schedule one complete model/Tool turn and return its cancellation handle.""" + if self.current_turn is not None and not self.current_turn.done(): + raise AgentTurnActiveError(f"Thread {self.id} already has an active Turn") + self.current_turn = asyncio.create_task( + self._run_turn(input), + name=f"agent-thread-{self.id}", + ) + return self.current_turn + + async def _run_turn(self, input: UserMessage) -> TurnTermination: + self._turn_index += 1 + self._model_calls = 0 + with operation( + "agent.turn", + attributes={ + "gen_ai.conversation.id": str(self.id), + "inkcre.agent.turn": self._turn_index, + }, + ) as observation: + span = observation.span + started = time.monotonic() + await trace( + "agent.turn.started", + self.id, + turn=self._turn_index, + input=input, + model=self.model, + max_model_calls=self.max_model_calls_per_turn, + ) + try: + outcome = await self._execute_turn(input) + except asyncio.CancelledError: + span.set_attribute("inkcre.agent.outcome", "cancelled") + await trace( + "agent.turn.finished", + self.id, + turn=self._turn_index, + model_calls=self._model_calls, + outcome="cancelled", + elapsed_seconds=time.monotonic() - started, + ) + raise + except Exception as error: + span.set_attribute("inkcre.agent.outcome", "failed") + await trace( + "agent.turn.finished", + self.id, + turn=self._turn_index, + model_calls=self._model_calls, + outcome="failed", + error_type=type(error).__name__, + error=str(error), + traceback=traceback.format_exc(), + elapsed_seconds=time.monotonic() - started, + ) + raise + await trace( + "agent.turn.finished", + self.id, + turn=self._turn_index, + model_calls=self._model_calls, + outcome=outcome, + elapsed_seconds=time.monotonic() - started, + ) + span.set_attribute("inkcre.agent.outcome", outcome.value) + span.set_attribute("inkcre.agent.model_calls", self._model_calls) + return outcome + + async def _execute_turn(self, input: UserMessage) -> TurnTermination: + self._state = await self._persistence.discard_trailing_incomplete_tool_calls(self.id) + self._state = await self._persistence.append(self.id, (input,)) + + while True: + self._model_calls += 1 + with operation( + "agent.step", + attributes={ + "gen_ai.conversation.id": str(self.id), + "inkcre.agent.turn": self._turn_index, + "inkcre.agent.model_call": self._model_calls, + }, + ): + await trace( + "agent.model.started", + self.id, + turn=self._turn_index, + call=self._model_calls, + ) + started = time.monotonic() + assistant = await AIManager.chat( + self._state.model, + self._state.messages, + self._state.tools, + self._state.tool_choice, + ) + await trace( + "agent.model.completed", + self.id, + turn=self._turn_index, + call=self._model_calls, + response=assistant, + elapsed_seconds=time.monotonic() - started, + ) + if not assistant.tool_calls: + self._state = await self._persistence.append(self.id, (assistant,)) + return TurnTermination.COMPLETED + + results = await self._execute_tool_batch(assistant) + self._state = await self._persistence.append( + self.id, + (assistant, ToolResultMessage(results=results)), + ) + if self._model_calls >= self._state.max_model_calls_per_turn: + return TurnTermination.MAX_MODEL_CALLS + + async def _execute_tool_batch( + self, + assistant: AssistantMessage, + ) -> tuple[ToolResult, ...]: + tasks = tuple( + asyncio.create_task( + self._execute_tool_call(call), + name=f"agent-tool-{call.tool}-{call.id}", + ) + for call in assistant.tool_calls + ) + return tuple(await asyncio.gather(*tasks)) + + async def _execute_tool_call(self, call: ToolCall) -> ToolResult: + with operation( + "agent.tool", + attributes={ + "gen_ai.operation.name": "execute_tool", + "gen_ai.conversation.id": str(self.id), + "inkcre.agent.turn": self._turn_index, + "inkcre.agent.model_call": self._model_calls, + # Provider-generated IDs may contain content; retain correlation as a digest. + "inkcre.agent.tool_call.id_hash": hashlib.sha256( + call.id.encode(errors="replace") + ).hexdigest(), + }, + ) as observation: + span = observation.span + tool = self._tools.get(call.tool) + if tool is not None: + span.set_attribute("gen_ai.tool.name", tool.definition.id) + await trace( + "agent.tool.started", + self.id, + turn=self._turn_index, + call=self._model_calls, + tool_call=call, + ) + started = time.monotonic() + try: + result = await self._invoke_tool_call(call) + except asyncio.CancelledError: + span.set_attribute("inkcre.agent.outcome", "cancelled") + await trace( + "agent.tool.cancelled", + self.id, + turn=self._turn_index, + call=self._model_calls, + tool_call_id=call.id, + tool=call.tool, + elapsed_seconds=time.monotonic() - started, + ) + raise + await trace( + "agent.tool.completed", + self.id, + turn=self._turn_index, + call=self._model_calls, + tool=call.tool, + result=result, + elapsed_seconds=time.monotonic() - started, + ) + span.set_attribute( + "inkcre.agent.outcome", "error" if result.is_error else "completed" + ) + if result.is_error: + span.set_attribute("error.type", "ToolResultError") + observation.outcome = "error" + return result + + async def _invoke_tool_call(self, call: ToolCall) -> ToolResult: + tool = self._tools.get(call.tool) + if tool is None: + return ToolResult( + tool_call_id=call.id, + content={"error": "unknown_tool", "tool": call.tool}, + is_error=True, + ) + + try: + tool_input = tool.input_model.model_validate(call.arguments) + except pydantic.ValidationError as error: + return ToolResult( + tool_call_id=call.id, + content=typing.cast(JSONValue, json.loads(error.json())), + is_error=True, + ) + + try: + content = tool.handler(tool_input) + if inspect.isawaitable(content): + content = await content + return ToolResult(tool_call_id=call.id, content=content) + except ToolExecutionError as error: + return ToolResult( + tool_call_id=call.id, + content=error.content, + is_error=True, + ) + except Exception as error: + await trace( + "agent.tool.exception", + self.id, + turn=self._turn_index, + call=self._model_calls, + tool_call_id=call.id, + tool=call.tool, + error_type=type(error).__name__, + error=str(error), + traceback=traceback.format_exc(), + ) + logger.exception("Unexpected Agent Tool failure", extra={"tool": call.tool}) + return ToolResult( + tool_call_id=call.id, + content={"error": "tool_execution_failed"}, + is_error=True, + ) diff --git a/inkcre_core/business/ai/__init__.py b/inkcre_core/business/ai/__init__.py new file mode 100644 index 00000000..df830957 --- /dev/null +++ b/inkcre_core/business/ai/__init__.py @@ -0,0 +1,45 @@ +"""AI registry facts and peer-local capability execution.""" + +from .contracts import ( + AICapabilityUnavailableError, + AIDialectAdapter, + AIExecutionRequirement, + AIFeatureUnavailableError, + AIInputUnavailableError, + AIModelDisabledError, + AIModelNotFoundError, + AIOutputContractError, + AIProviderDisabledError, + AIProviderNotFoundError, + DuplicateAIDialectRegistrationError, + InvalidAIProviderConfigError, + UnknownAIDialectError, +) +from .main import AIManager + +# Import installed core dialects only after AIManager exists so decorators register. +from .dialects import ( + AlibabaModelStudioDialect, + OpenAICompatibleConfig, + OpenAICompatibleDialect, +) + +__all__ = [ + "AICapabilityUnavailableError", + "AIDialectAdapter", + "AIExecutionRequirement", + "AIFeatureUnavailableError", + "AIInputUnavailableError", + "AIManager", + "AIModelDisabledError", + "AIModelNotFoundError", + "AIOutputContractError", + "AIProviderDisabledError", + "AIProviderNotFoundError", + "AlibabaModelStudioDialect", + "DuplicateAIDialectRegistrationError", + "InvalidAIProviderConfigError", + "OpenAICompatibleConfig", + "OpenAICompatibleDialect", + "UnknownAIDialectError", +] diff --git a/inkcre_core/business/ai/contracts.py b/inkcre_core/business/ai/contracts.py new file mode 100644 index 00000000..a1d4eac4 --- /dev/null +++ b/inkcre_core/business/ai/contracts.py @@ -0,0 +1,111 @@ +"""AI dialect adapter boundary and stable execution failures.""" + +import abc +from collections.abc import Sequence +from dataclasses import dataclass + +import pydantic + +from inkcre_core.schemas.ai import ( + AICapabilityType, + AssistantMessage, + FunctionTool, + Message, + ToolChoice, +) +from inkcre_core.schemas.info_base.main import Vector + + +@dataclass(frozen=True) +class AIExecutionRequirement: + """Static capability shape used for peer-local execution eligibility.""" + + capability: AICapabilityType + input_modalities: frozenset[str] + output_modalities: frozenset[str] + features: frozenset[str] = frozenset() + tool_choice: ToolChoice | None = None + + +class DuplicateAIDialectRegistrationError(ValueError): + """Two adapter implementations claimed the same exact dialect ID.""" + + +class UnknownAIDialectError(LookupError): + """The current peer has no adapter for an exact persisted dialect ID.""" + + +class AIProviderNotFoundError(LookupError): + """A requested AI model references no provider visible to this peer.""" + + +class AIModelNotFoundError(LookupError): + """A requested AI model does not exist.""" + + +class AIProviderDisabledError(RuntimeError): + """A disabled provider rejected a new operation.""" + + +class AIModelDisabledError(RuntimeError): + """A disabled model rejected a new operation.""" + + +class InvalidAIProviderConfigError(RuntimeError): + """Persisted provider config does not satisfy its dialect contract.""" + + +class AICapabilityUnavailableError(RuntimeError): + """The selected provider-bound model does not declare an operation.""" + + +class AIFeatureUnavailableError(RuntimeError): + """Effective model/adapter support lacks a requested feature.""" + + +class AIInputUnavailableError(RuntimeError): + """A canonical input cannot be conveyed through the selected dialect.""" + + +class AIOutputContractError(RuntimeError): + """A provider response violated the canonical capability result contract.""" + + +class AIDialectAdapter(abc.ABC): + """Graph-blind provider config/client construction and wire translation.""" + + supported_features: dict[AICapabilityType, frozenset[str]] = {} + supported_input_modalities: dict[AICapabilityType, frozenset[str]] = {} + + @abc.abstractmethod + async def embed( + self, + config: pydantic.BaseModel, + native_model_id: str, + inputs: Sequence[str], + dimensions: int, + ) -> tuple[Vector, ...]: ... + + @abc.abstractmethod + async def chat( + self, + config: pydantic.BaseModel, + native_model_id: str, + messages: Sequence[Message], + tools: Sequence[FunctionTool], + tool_choice: ToolChoice | None, + ) -> AssistantMessage: ... + + def supports_feature(self, capability: AICapabilityType, feature: str) -> bool: + return feature in self.supported_features.get(capability, frozenset()) + + def supports_input_modality( + self, + capability: AICapabilityType, + modality: str, + ) -> bool: + return modality in self.supported_input_modalities.get(capability, frozenset()) + + def supports_tool_choice(self, tool_choice: ToolChoice) -> bool: + del tool_choice + return False diff --git a/inkcre_core/business/ai/dialects/__init__.py b/inkcre_core/business/ai/dialects/__init__.py new file mode 100644 index 00000000..93f92271 --- /dev/null +++ b/inkcre_core/business/ai/dialects/__init__.py @@ -0,0 +1,10 @@ +"""Installed core AI dialect adapters.""" + +from .alibaba_model_studio import AlibabaModelStudioDialect +from .openai_compatible import OpenAICompatibleConfig, OpenAICompatibleDialect + +__all__ = [ + "AlibabaModelStudioDialect", + "OpenAICompatibleConfig", + "OpenAICompatibleDialect", +] diff --git a/inkcre_core/business/ai/dialects/alibaba_model_studio.py b/inkcre_core/business/ai/dialects/alibaba_model_studio.py new file mode 100644 index 00000000..48512bc6 --- /dev/null +++ b/inkcre_core/business/ai/dialects/alibaba_model_studio.py @@ -0,0 +1,181 @@ +"""Alibaba Model Studio chat dialect, including native multimodal parts.""" + +from collections.abc import Sequence +import json +import typing +import time + +from openai import AsyncStream +from openai.types.chat import ChatCompletionChunk +from opentelemetry import trace +from opentelemetry.trace import INVALID_SPAN +import pydantic + +from inkcre_core.schemas.ai import ( + AudioContentPart, + AssistantMessage, + FunctionTool, + ImageContentPart, + Message, + TextContentPart, + ToolCall, + ToolChoice, + UserMessage, + VideoContentPart, +) + +from inkcre_core.libs.obsrv.telemetry import is_enabled + +from ..contracts import AIOutputContractError +from ..main import AIManager +from ..telemetry import record_response_usage +from .openai_compatible import OpenAICompatibleConfig, OpenAICompatibleDialect + + +@AIManager.register_dialect( + "core.alibaba-model-studio.v1", + description="Alibaba Model Studio OpenAI-compatible multimodal chat protocol.", + config_model=OpenAICompatibleConfig, +) +class AlibabaModelStudioDialect(OpenAICompatibleDialect): + """Translate canonical chat through Model Studio's multimodal extensions.""" + + supported_features = {"chat": frozenset({"tool_calling"})} + supported_input_modalities = { + "embedding": frozenset({"text"}), + "chat": frozenset({"text", "image", "audio", "video"}), + } + + @classmethod + def _user_content_params(cls, message: UserMessage) -> list[dict[str, typing.Any]]: + result: list[dict[str, typing.Any]] = [] + for part in message.content: + if isinstance(part, TextContentPart): + result.append({"type": "text", "text": part.text}) + elif isinstance(part, ImageContentPart): + result.append( + { + "type": "image_url", + "image_url": { + "url": cls._url_or_inline_data( + mime_type=part.mime_type, + data=part.data, + transfer_url=part.transfer_url, + ) + }, + } + ) + elif isinstance(part, AudioContentPart): + result.append( + { + "type": "input_audio", + "input_audio": { + "data": cls._url_or_inline_data( + mime_type=part.mime_type, + data=part.data, + transfer_url=part.transfer_url, + ), + "format": part.mime_type.removeprefix("audio/"), + }, + } + ) + elif isinstance(part, VideoContentPart): + result.append( + { + "type": "video_url", + "video_url": { + "url": cls._url_or_inline_data( + mime_type=part.mime_type, + data=part.data, + transfer_url=part.transfer_url, + ) + }, + } + ) + return result + + async def chat( + self, + config: pydantic.BaseModel, + native_model_id: str, + messages: Sequence[Message], + tools: Sequence[FunctionTool], + tool_choice: ToolChoice | None, + ) -> AssistantMessage: + client = self._client_factory(self._config(config)) + arguments: dict[str, typing.Any] = { + "model": native_model_id, + "messages": self._message_params(messages), + "stream": True, + "stream_options": {"include_usage": True}, + "modalities": ["text"], + } + if tools: + arguments["tools"] = self._tool_params(tools) + if tool_choice is not None: + arguments["tool_choice"] = self._tool_choice_param(tool_choice) + + span = trace.get_current_span() if is_enabled() else INVALID_SPAN + span.set_attribute("gen_ai.request.stream", True) + started = time.monotonic() + usage = None + finish_reasons: list[str] = [] + first_chunk = True + try: + stream = typing.cast( + AsyncStream[ChatCompletionChunk], + await client.chat.completions.create(**arguments), + ) + text_parts: list[str] = [] + calls: dict[int, dict[str, str]] = {} + async for chunk in stream: + if first_chunk: + span.set_attribute( + "gen_ai.response.time_to_first_chunk", time.monotonic() - started + ) + first_chunk = False + # Usage-only terminal chunks have no choices. Totals replace earlier totals. + if chunk.usage is not None: + usage = chunk.usage + if not chunk.choices: + continue + if chunk.choices[0].finish_reason is not None: + finish_reasons = [chunk.choices[0].finish_reason] + delta = chunk.choices[0].delta + if isinstance(delta.content, str): + text_parts.append(delta.content) + for call in delta.tool_calls or (): + current = calls.setdefault(call.index, {"id": "", "name": "", "arguments": ""}) + if call.id: + current["id"] = call.id + if call.function is not None: + if call.function.name: + current["name"] += call.function.name + if call.function.arguments: + current["arguments"] += call.function.arguments + finally: + record_response_usage("chat", usage, finish_reasons) + await client.close() + + parsed_calls: list[ToolCall] = [] + for index in sorted(calls): + call = calls[index] + if not call["id"] or not call["name"]: + raise AIOutputContractError("Alibaba chat stream returned an incomplete ToolCall") + try: + arguments_object = json.loads(call["arguments"]) + except json.JSONDecodeError as error: + raise AIOutputContractError( + f"ToolCall {call['id']!r} arguments are not valid JSON" + ) from error + if not isinstance(arguments_object, dict): + raise AIOutputContractError( + f"ToolCall {call['id']!r} arguments must be a JSON object" + ) + parsed_calls.append( + ToolCall(id=call["id"], tool=call["name"], arguments=arguments_object) + ) + content = "".join(text_parts) or None + if content is None and not parsed_calls: + raise AIOutputContractError("Alibaba chat stream returned no content or ToolCalls") + return AssistantMessage(content=content, tool_calls=tuple(parsed_calls)) diff --git a/inkcre_core/business/ai/dialects/openai_compatible.py b/inkcre_core/business/ai/dialects/openai_compatible.py new file mode 100644 index 00000000..f1882df6 --- /dev/null +++ b/inkcre_core/business/ai/dialects/openai_compatible.py @@ -0,0 +1,323 @@ +"""OpenAI-compatible AI dialect across embedding and chat capabilities.""" + +import base64 +from collections.abc import Callable, Sequence +import json +import typing + +from openai import AsyncOpenAI +from openai.types.chat import ChatCompletion, ChatCompletionMessageParam +import pydantic + +from inkcre_core.schemas.ai import ( + AudioContentPart, + AssistantMessage, + FunctionTool, + ImageContentPart, + Message, + NamedToolChoice, + SystemMessage, + TextContentPart, + ToolCall, + ToolChoice, + ToolResultMessage, + UserMessage, + VideoContentPart, +) +from inkcre_core.schemas.info_base.main import Vector + +from ..contracts import AIDialectAdapter, AIInputUnavailableError, AIOutputContractError +from ..main import AIManager +from ..telemetry import record_response_usage + +INLINE_MEDIA_MAX_BYTES = 7 * 1024 * 1024 + + +class OpenAICompatibleConfig(pydantic.BaseModel): + """Connection values for one OpenAI-compatible provider instance.""" + + model_config = pydantic.ConfigDict(extra="forbid") + + api_key: pydantic.SecretStr + base_url: str | None = pydantic.Field(default=None, min_length=1) + + +OpenAIClientFactory: typing.TypeAlias = Callable[[OpenAICompatibleConfig], AsyncOpenAI] + + +@AIManager.register_dialect( + "core.openai-compatible.v1", + description="OpenAI-compatible embedding and chat protocol.", + config_model=OpenAICompatibleConfig, +) +class OpenAICompatibleDialect(AIDialectAdapter): + """Translate canonical AI contracts through the OpenAI Python SDK.""" + + supported_features = {"chat": frozenset({"tool_calling"})} + supported_input_modalities = { + "embedding": frozenset({"text"}), + "chat": frozenset({"text", "image", "audio"}), + } + + def __init__(self, client_factory: OpenAIClientFactory | None = None) -> None: + self._client_factory = client_factory or self._create_client + + @staticmethod + def _create_client(config: OpenAICompatibleConfig) -> AsyncOpenAI: + return AsyncOpenAI( + api_key=config.api_key.get_secret_value(), + base_url=config.base_url, + ) + + @staticmethod + def _config(config: pydantic.BaseModel) -> OpenAICompatibleConfig: + return typing.cast(OpenAICompatibleConfig, config) + + async def embed( + self, + config: pydantic.BaseModel, + native_model_id: str, + inputs: Sequence[str], + dimensions: int, + ) -> tuple[Vector, ...]: + client = self._client_factory(self._config(config)) + usage = None + try: + response = await client.embeddings.create( + model=native_model_id, + input=tuple(inputs), + dimensions=dimensions, + encoding_format="float", + ) + usage = getattr(response, "usage", None) + finally: + record_response_usage("embeddings", usage) + await client.close() + + by_index = {item.index: tuple(item.embedding) for item in response.data} + expected = set(range(len(inputs))) + if set(by_index) != expected: + raise AIOutputContractError( + "OpenAI-compatible embedding response indices do not cover the input batch" + ) + return tuple(by_index[index] for index in range(len(inputs))) + + @staticmethod + def _data_url(mime_type: str, data: bytes) -> str: + encoded = base64.b64encode(data).decode("ascii") + return f"data:{mime_type};base64,{encoded}" + + @classmethod + def _url_or_inline_data( + cls, + *, + mime_type: str, + data: bytes, + transfer_url: str | None, + ) -> str: + if len(data) <= INLINE_MEDIA_MAX_BYTES: + return cls._data_url(mime_type, data) + if transfer_url is not None: + return transfer_url + raise AIInputUnavailableError( + "Media exceeds the inline transfer bound and has no transfer URL" + ) + + @staticmethod + def _audio_format(mime_type: str) -> str: + formats = { + "audio/mpeg": "mp3", + "audio/mp3": "mp3", + "audio/wav": "wav", + "audio/x-wav": "wav", + } + try: + return formats[mime_type] + except KeyError as error: + raise AIInputUnavailableError( + f"OpenAI Chat input_audio does not support {mime_type!r}" + ) from error + + @classmethod + def _user_content_params(cls, message: UserMessage) -> list[dict[str, typing.Any]]: + result: list[dict[str, typing.Any]] = [] + for part in message.content: + if isinstance(part, TextContentPart): + result.append({"type": "text", "text": part.text}) + elif isinstance(part, ImageContentPart): + result.append( + { + "type": "image_url", + "image_url": { + "url": cls._url_or_inline_data( + mime_type=part.mime_type, + data=part.data, + transfer_url=part.transfer_url, + ) + }, + } + ) + elif isinstance(part, AudioContentPart): + if len(part.data) > INLINE_MEDIA_MAX_BYTES: + raise AIInputUnavailableError( + "OpenAI Chat input_audio cannot use a transfer URL and exceeds the " + "inline transfer bound" + ) + result.append( + { + "type": "input_audio", + "input_audio": { + "data": base64.b64encode(part.data).decode("ascii"), + "format": cls._audio_format(part.mime_type), + }, + } + ) + elif isinstance(part, VideoContentPart): + raise AIInputUnavailableError( + "OpenAI Chat does not define a canonical video input part" + ) + return result + + @classmethod + def _message_params(cls, messages: Sequence[Message]) -> list[ChatCompletionMessageParam]: + result: list[ChatCompletionMessageParam] = [] + for message in messages: + if isinstance(message, SystemMessage): + result.append({"role": "system", "content": message.content}) + elif isinstance(message, UserMessage): + result.append( + typing.cast( + ChatCompletionMessageParam, + {"role": "user", "content": cls._user_content_params(message)}, + ) + ) + elif isinstance(message, AssistantMessage): + result.append( + { + "role": "assistant", + "content": message.content, + "tool_calls": [ + { + "id": call.id, + "type": "function", + "function": { + "name": call.tool, + "arguments": json.dumps( + call.arguments, + separators=(",", ":"), + sort_keys=True, + ), + }, + } + for call in message.tool_calls + ], + } + ) + elif isinstance(message, ToolResultMessage): + result.extend( + { + "role": "tool", + "tool_call_id": tool_result.tool_call_id, + "content": json.dumps( + { + "content": tool_result.content, + "is_error": tool_result.is_error, + }, + separators=(",", ":"), + sort_keys=True, + ), + } + for tool_result in message.results + ) + return result + + @staticmethod + def _tool_params(tools: Sequence[FunctionTool]) -> list[dict[str, typing.Any]]: + return [ + { + "type": "function", + "function": { + "name": tool.id, + "description": tool.description, + "parameters": tool.input_schema, + }, + } + for tool in tools + ] + + @staticmethod + def _tool_choice_param(tool_choice: ToolChoice) -> typing.Any: + if isinstance(tool_choice, NamedToolChoice): + return { + "type": "function", + "function": {"name": tool_choice.tool}, + } + return tool_choice + + def supports_tool_choice(self, tool_choice: ToolChoice) -> bool: + return isinstance(tool_choice, NamedToolChoice) or tool_choice in { + "none", + "auto", + "required", + } + + @staticmethod + def _assistant_message(message: typing.Any) -> AssistantMessage: + calls: list[ToolCall] = [] + for call in message.tool_calls or (): + if call.type != "function": + raise AIOutputContractError( + f"Unsupported OpenAI-compatible ToolCall type: {call.type!r}" + ) + try: + parsed_arguments = json.loads(call.function.arguments) + except (TypeError, json.JSONDecodeError) as error: + raise AIOutputContractError( + f"ToolCall {call.id!r} arguments are not valid JSON" + ) from error + if not isinstance(parsed_arguments, dict): + raise AIOutputContractError(f"ToolCall {call.id!r} arguments must be a JSON object") + calls.append( + ToolCall( + id=call.id, + tool=call.function.name, + arguments=parsed_arguments, + ) + ) + return AssistantMessage(content=message.content, tool_calls=tuple(calls)) + + async def chat( + self, + config: pydantic.BaseModel, + native_model_id: str, + messages: Sequence[Message], + tools: Sequence[FunctionTool], + tool_choice: ToolChoice | None, + ) -> AssistantMessage: + client = self._client_factory(self._config(config)) + arguments: dict[str, typing.Any] = { + "model": native_model_id, + "messages": self._message_params(messages), + "stream": False, + } + if tools: + arguments["tools"] = self._tool_params(tools) + if tool_choice is not None: + arguments["tool_choice"] = self._tool_choice_param(tool_choice) + + usage = None + finish_reasons: list[str] = [] + try: + response = typing.cast( + ChatCompletion, + await client.chat.completions.create(**arguments), + ) + usage = response.usage + finish_reasons = [choice.finish_reason for choice in response.choices] + finally: + record_response_usage("chat", usage, finish_reasons) + await client.close() + + if not response.choices: + raise AIOutputContractError("OpenAI-compatible chat response has no choices") + return self._assistant_message(response.choices[0].message) diff --git a/inkcre_core/business/ai/main.py b/inkcre_core/business/ai/main.py new file mode 100644 index 00000000..2cfae318 --- /dev/null +++ b/inkcre_core/business/ai/main.py @@ -0,0 +1,396 @@ +"""Shared AI fact management and peer-local capability routing.""" + +from collections.abc import Sequence +from dataclasses import dataclass +import math +import typing + +import pydantic +from opentelemetry.trace import SpanKind + +from inkcre_core.database_contract.profile import BUILTIN_AI_DIALECTS_BY_ID +from inkcre_core.schemas.ai import ( + AICapabilityType, + AIDialectID, + AIModelCapability, + AIModelID, + AIModelModel, + AIProviderModel, + AssistantMessage, + FunctionTool, + ImageContentPart, + Message, + AudioContentPart, + TextContentPart, + ToolChoice, + UserMessage, + VideoContentPart, + validate_message_history, +) +from inkcre_core.schemas.info_base.main import Vector + +from inkcre_core.persistence.ai.uow import ai_uow +from inkcre_core.libs.obsrv.telemetry import operation + +from .telemetry import model_attributes + +from .contracts import ( + AICapabilityUnavailableError, + AIDialectAdapter, + AIExecutionRequirement, + AIFeatureUnavailableError, + AIInputUnavailableError, + AIModelDisabledError, + AIModelNotFoundError, + AIOutputContractError, + AIProviderDisabledError, + AIProviderNotFoundError, + DuplicateAIDialectRegistrationError, + InvalidAIProviderConfigError, + UnknownAIDialectError, +) + + +ConfigModelT = typing.TypeVar("ConfigModelT", bound=pydantic.BaseModel) +AdapterT = typing.TypeVar("AdapterT", bound=type[AIDialectAdapter]) + + +@dataclass(frozen=True) +class _DialectRegistration: + adapter: AIDialectAdapter + adapter_type: type[AIDialectAdapter] + config_model: type[pydantic.BaseModel] + description: str + + +@dataclass(frozen=True) +class _ExecutionTarget: + model: AIModelModel + provider: AIProviderModel + adapter: AIDialectAdapter + config: pydantic.BaseModel + + +class AIManager: + """Sole domain manager for AI dialects, shared facts and execution.""" + + _DIALECTS: dict[AIDialectID, _DialectRegistration] = {} + + @classmethod + async def get_model(cls, model_id: AIModelID) -> AIModelModel | None: + """Read a model without requiring its provider/dialect to execute here.""" + async with ai_uow() as models: + return await models.get_model(model_id) + + @classmethod + async def list_models( + cls, *, limit: int | None = None, cursor: int | None = None + ) -> tuple[list[AIModelModel], int | None]: + async with ai_uow() as models: + return await models.list_models(limit=limit, cursor=cursor) + + @classmethod + def register_dialect( + cls, + dialect_id: AIDialectID, + *, + description: str, + config_model: type[ConfigModelT], + ) -> typing.Callable[[AdapterT], AdapterT]: + """Bind one adapter class to an exact ID and return it unchanged.""" + + def decorator(adapter_type: AdapterT) -> AdapterT: + existing = cls._DIALECTS.get(dialect_id) + if existing is not None: + if ( + existing.adapter_type is adapter_type + and existing.config_model is config_model + and existing.description == description + ): + return adapter_type + raise DuplicateAIDialectRegistrationError( + f"AI dialect {dialect_id!r} is already registered by " + f"{existing.adapter_type.__qualname__}" + ) + cls._DIALECTS[dialect_id] = _DialectRegistration( + adapter=adapter_type(), + adapter_type=adapter_type, + config_model=config_model, + description=description, + ) + return adapter_type + + return decorator + + @classmethod + async def sync_dialects_async(cls) -> None: + records = [] + for dialect_id, registration in cls._DIALECTS.items(): + builtin = BUILTIN_AI_DIALECTS_BY_ID.get(dialect_id) + records.append( + { + "id": dialect_id, + "description": builtin.description + if builtin is not None + else registration.description, + "config_schema": builtin.config_schema + if builtin is not None + else registration.config_model.model_json_schema(), + } + ) + async with ai_uow() as models: + await models.sync_dialects(records) + + @classmethod + def _registration(cls, dialect_id: AIDialectID) -> _DialectRegistration: + try: + return cls._DIALECTS[dialect_id] + except KeyError as error: + raise UnknownAIDialectError( + f"No local adapter implements AI dialect {dialect_id!r}" + ) from error + + @classmethod + async def _load_target_async(cls, model_id: AIModelID) -> _ExecutionTarget: + async with ai_uow() as models: + model = await models.get_model(model_id) + provider = None if model is None else await models.get_provider(model.provider) + return cls._execution_target(model_id, model, provider) + + @classmethod + def _execution_target( + cls, model_id: AIModelID, model: AIModelModel | None, provider: AIProviderModel | None + ) -> _ExecutionTarget: + if model is None: + raise AIModelNotFoundError(f"AI model {model_id} does not exist") + if not model.enabled: + raise AIModelDisabledError(f"AI model {model_id} is disabled") + if provider is None: + raise AIProviderNotFoundError( + f"AI model {model_id} references missing provider {model.provider}" + ) + if not provider.enabled: + raise AIProviderDisabledError(f"AI provider {provider.id} is disabled") + registration = cls._registration(provider.dialect) + try: + config = registration.config_model.model_validate(provider.config) + except pydantic.ValidationError as error: + raise InvalidAIProviderConfigError( + f"AI provider {provider.id} does not satisfy dialect {provider.dialect!r}" + ) from error + return _ExecutionTarget(model, provider, registration.adapter, config) + + @staticmethod + def _capability( + model: AIModelModel, + capability_type: AICapabilityType, + ) -> AIModelCapability: + capability = next( + (item for item in model.capabilities if item.type == capability_type), + None, + ) + if capability is None: + raise AICapabilityUnavailableError( + f"AI model {model.id} does not declare {capability_type!r}" + ) + return capability + + @staticmethod + def _require_modalities( + model: AIModelModel, + capability: AIModelCapability, + *, + input_: str, + output: str, + ) -> None: + if ( + input_ not in capability.input_modalities + or output not in capability.output_modalities + ): + raise AICapabilityUnavailableError( + f"AI model {model.id} {capability.type!r} does not support {input_!r} -> {output!r}" + ) + + @classmethod + async def can_execute( + cls, + model: AIModelID, + requirement: AIExecutionRequirement, + ) -> bool: + """Return static peer-local eligibility without probing a remote provider.""" + try: + target = await cls._load_target_async(model) + capability = cls._capability(target.model, requirement.capability) + except ( + AIModelNotFoundError, + AIModelDisabledError, + AIProviderNotFoundError, + AIProviderDisabledError, + UnknownAIDialectError, + InvalidAIProviderConfigError, + AICapabilityUnavailableError, + ): + return False + requested_inputs = requirement.input_modalities + requested_outputs = requirement.output_modalities + requested_features = requirement.features + if not requested_inputs <= set(capability.input_modalities): + return False + if not requested_outputs <= set(capability.output_modalities): + return False + if not requested_features <= set(capability.features): + return False + if any( + not target.adapter.supports_input_modality(requirement.capability, modality) + for modality in requested_inputs + ): + return False + if any( + not target.adapter.supports_feature(requirement.capability, feature) + for feature in requested_features + ): + return False + return requirement.tool_choice is None or target.adapter.supports_tool_choice( + requirement.tool_choice + ) + + @classmethod + async def embed( + cls, + model: AIModelID, + inputs: Sequence[str], + dimensions: int, + ) -> tuple[Vector, ...]: + """Embed one ordered non-empty text batch through the selected model.""" + with operation( + "ai.embed", + kind=SpanKind.CLIENT, + attributes={ + "gen_ai.operation.name": "embeddings", + "inkcre.ai.model.id": model, + "inkcre.ai.usage.input.source": "unavailable", + "inkcre.ai.usage.output.source": "unavailable", + }, + ) as observation: + span = observation.span + if not inputs: + raise ValueError("embedding inputs must not be empty") + if dimensions <= 0: + raise ValueError("embedding dimensions must be positive") + target = await cls._load_target_async(model) + span.set_attributes( + model_attributes( + target.model.native_model_id, target.provider.dialect, target.model.provider + ) + ) + capability = cls._capability(target.model, "embedding") + cls._require_modalities( + target.model, + capability, + input_="text", + output="vector", + ) + vectors = await target.adapter.embed( + target.config, + target.model.native_model_id, + tuple(inputs), + dimensions, + ) + if len(vectors) != len(inputs): + raise AIOutputContractError( + f"Embedding result count {len(vectors)} does not match input count {len(inputs)}" + ) + for vector in vectors: + if len(vector) != dimensions: + raise AIOutputContractError( + f"Embedding dimension {len(vector)} does not match requested {dimensions}" + ) + if not vector or not all(math.isfinite(value) for value in vector): + raise AIOutputContractError("Embedding vectors must be finite and non-empty") + if not any(value != 0 for value in vector): + raise AIOutputContractError("Embedding vectors must be non-zero") + return vectors + + @classmethod + async def chat( + cls, + model: AIModelID, + messages: Sequence[Message], + tools: Sequence[FunctionTool] = (), + tool_choice: ToolChoice | None = None, + ) -> AssistantMessage: + """Execute one provider-neutral chat model call without owning history.""" + with operation( + "ai.chat", + kind=SpanKind.CLIENT, + attributes={ + "gen_ai.operation.name": "chat", + "inkcre.ai.model.id": model, + "inkcre.ai.usage.input.source": "unavailable", + "inkcre.ai.usage.output.source": "unavailable", + }, + ) as observation: + span = observation.span + history = validate_message_history(messages) + target = await cls._load_target_async(model) + span.set_attributes( + model_attributes( + target.model.native_model_id, target.provider.dialect, target.model.provider + ) + ) + capability = cls._capability(target.model, "chat") + # System instructions, Tool schemas/results and Assistant history all travel + # through the textual chat channel even when the latest User turn is media-only. + input_modalities: set[str] = {"text"} + for message in history: + if not isinstance(message, UserMessage): + continue + for part in message.content: + if isinstance(part, TextContentPart): + input_modalities.add("text") + elif isinstance(part, ImageContentPart): + input_modalities.add("image") + elif isinstance(part, AudioContentPart): + input_modalities.add("audio") + elif isinstance(part, VideoContentPart): + input_modalities.add("video") + for modality in sorted(input_modalities): + cls._require_modalities( + target.model, + capability, + input_=modality, + output="text", + ) + if not target.adapter.supports_input_modality("chat", modality): + raise AIInputUnavailableError( + f"AI dialect {target.provider.dialect!r} cannot convey " + f"chat modality {modality!r}" + ) + + tool_ids = tuple(tool.id for tool in tools) + if len(tool_ids) != len(set(tool_ids)): + raise ValueError("Tool IDs must be unique") + requires_tool_calling = bool(tools) or tool_choice is not None + if requires_tool_calling: + if "tool_calling" not in capability.features or not target.adapter.supports_feature( + "chat", "tool_calling" + ): + raise AIFeatureUnavailableError( + f"AI model {model} and dialect {target.provider.dialect!r} do not jointly " + "support tool_calling" + ) + if tool_choice is not None: + if not tools: + raise ValueError("tool_choice requires at least one Tool") + if not target.adapter.supports_tool_choice(tool_choice): + raise AIFeatureUnavailableError( + f"AI dialect {target.provider.dialect!r} cannot represent tool_choice" + ) + + return await target.adapter.chat( + target.config, + target.model.native_model_id, + history, + tuple(tools), + tool_choice, + ) diff --git a/inkcre_core/business/ai/telemetry.py b/inkcre_core/business/ai/telemetry.py new file mode 100644 index 00000000..46830bde --- /dev/null +++ b/inkcre_core/business/ai/telemetry.py @@ -0,0 +1,86 @@ +"""Content-free mapping for the pinned GenAI semantic conventions. + +Keys follow semantic-conventions-genai e07f4ebacb08f56db8c4c882d117720333fbca04. +Provider responses are untrusted: copy numeric usage and known finish reasons only. +""" + +from collections.abc import Sequence +import re + +from openai.types.completion_usage import CompletionUsage +from openai.types.create_embedding_response import Usage as EmbeddingUsage +from opentelemetry import trace +from opentelemetry.util.types import AttributeValue + +from inkcre_core.libs.obsrv.telemetry import is_enabled, record_ai_usage + + +_IDENTIFIER = re.compile(r"[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}\Z") +_FINISH_REASONS = frozenset( + {"stop", "length", "tool_calls", "content_filter", "function_call"} +) + + +def model_attributes( + native_model_id: str, dialect: str, provider_id: int +) -> dict[str, AttributeValue]: + attributes: dict[str, AttributeValue] = { + "inkcre.ai.provider.id": provider_id, + "inkcre.ai.dialect": dialect if _IDENTIFIER.fullmatch(dialect) else "other", + "gen_ai.provider.name": { + "core.openai-compatible.v1": "openai", + "core.alibaba-model-studio.v1": "alibaba", + }.get(dialect, "other"), + } + # This is the configured model identifier, never the provider's arbitrary echo. + if _IDENTIFIER.fullmatch(native_model_id): + attributes["gen_ai.request.model"] = native_model_id + return attributes + + +def record_response_usage( + operation: str, + usage: CompletionUsage | EmbeddingUsage | None, + finish_reasons: Sequence[str] = (), +) -> None: + """Record one response's totals once; absent or invalid totals stay unknown.""" + if not is_enabled(): + return + span = trace.get_current_span() + input_tokens = getattr(usage, "prompt_tokens", None) + output_tokens = getattr(usage, "completion_tokens", None) + input_tokens = ( + input_tokens if type(input_tokens) is int and 0 <= input_tokens < 2**63 else None + ) + output_tokens = ( + output_tokens if type(output_tokens) is int and 0 <= output_tokens < 2**63 else None + ) + for direction, value in (("input", input_tokens), ("output", output_tokens)): + span.set_attribute( + f"inkcre.ai.usage.{direction}.source", + "provider" if value is not None else "unavailable", + ) + if value is not None: + span.set_attribute(f"gen_ai.usage.{direction}_tokens", value) + if isinstance(usage, CompletionUsage): + for key, value in ( + ( + "gen_ai.usage.cache_read.input_tokens", + getattr(usage.prompt_tokens_details, "cached_tokens", None), + ), + ( + "gen_ai.usage.reasoning.output_tokens", + getattr(usage.completion_tokens_details, "reasoning_tokens", None), + ), + ): + if type(value) is int and 0 <= value < 2**63: + span.set_attribute(key, value) + if finish_reasons: + span.set_attribute( + "gen_ai.response.finish_reasons", + tuple( + reason if isinstance(reason, str) and reason in _FINISH_REASONS else "other" + for reason in finish_reasons[:16] + ), + ) + record_ai_usage(operation, input_tokens=input_tokens, output_tokens=output_tokens) diff --git a/inkcre_core/business/cron.py b/inkcre_core/business/cron.py new file mode 100644 index 00000000..745a7542 --- /dev/null +++ b/inkcre_core/business/cron.py @@ -0,0 +1,234 @@ +"""Application-owned materialization of deployment-wide Cron occurrences.""" + +from __future__ import annotations + +import datetime +import typing +from zoneinfo import ZoneInfo, ZoneInfoNotFoundError + +import croniter # pyrefly: ignore[untyped-import] +import pydantic + +from inkcre_core.business.deployment_config import ( + DeploymentConfigManager, + DeploymentConfigService, +) +from inkcre_core.business.job import JobManager +from inkcre_core.persistence.cron.uow import cron_uow +from inkcre_core.schemas.cron import CronForm, CronID, CronModel, CronUpdateForm +from inkcre_core.schemas.job import JobModel, JobStatus +from inkcre_core.libs.obsrv.main import get_logger +from inkcre_core.libs.obsrv.telemetry import operation, emit_event +from inkcre_core.validation import input_path + + +CRON_CONFIG_KEY = "core.cron" +CRON_CONFIG_SCHEMA_ID = "core.cron.config.v1" +LOGGER = get_logger().getChild(__name__) + + +class CronNotFoundError(LookupError): + """A Cron template does not exist.""" + + +class InvalidCronScheduleError(ValueError): + """A submitted schedule is not a five-field UNIX Cron expression.""" + + +class CronDeploymentConfig(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid") + + timezone: str = "UTC" + + @pydantic.field_validator("timezone") + @classmethod + def validate_timezone(cls, value: str) -> str: + try: + ZoneInfo(value) + except ZoneInfoNotFoundError as error: + raise ValueError("timezone must be an installed IANA timezone") from error + return value + + +DeploymentConfigManager.register_schema( + CRON_CONFIG_SCHEMA_ID, CronDeploymentConfig, keys=(CRON_CONFIG_KEY,) +) + + +class InvalidCronTemplateError(ValueError): + """A failed template must leave its complete transaction before discovery continues.""" + + +class CronManager: + """Evaluate only the current minute and atomically create typed Jobs.""" + + @classmethod + async def _timezone(cls) -> ZoneInfo: + config = await DeploymentConfigService.get(CRON_CONFIG_KEY) + if config is None: + return ZoneInfo("UTC") + return ZoneInfo(typing.cast(CronDeploymentConfig, config).timezone) + + @classmethod + def _matches(cls, schedule: str, now: datetime.datetime) -> bool: + return croniter.croniter.match(schedule, now) + + @classmethod + async def check(cls) -> int: + """Materialize each matching occurrence in one independent transaction.""" + async with cron_uow() as uow: + cron_ids = await uow.crons.enabled_ids() + timezone = await cls._timezone() + created = 0 + for cron_id in cron_ids: + if cron_id is None: + continue + try: + created += await cls._materialize(cron_id, timezone) + except InvalidCronTemplateError: + LOGGER.exception("Cron Job template is invalid", extra={"cron_id": cron_id}) + return created + + @classmethod + async def _materialize(cls, cron_id: CronID, timezone: ZoneInfo) -> int: + with operation("cron.materialize", attributes={"inkcre.cron.id": cron_id}): + return await cls._materialize_occurrence(cron_id, timezone) + + @classmethod + async def _materialize_occurrence(cls, cron_id: CronID, timezone: ZoneInfo) -> int: + async with cron_uow() as uow: + cron = await uow.crons.get(cron_id, lock=True, skip_locked=True) + if cron is None or not cron.enabled: + return 0 + database_now = await uow.crons.database_now() + occurrence = database_now.astimezone(datetime.UTC).replace(second=0, microsecond=0) + local_minute = database_now.astimezone(timezone).replace(second=0, microsecond=0) + try: + matches = cls._matches(cron.schedule, local_minute) + except (ValueError, KeyError): + LOGGER.exception("Invalid persisted Cron schedule", extra={"cron_id": cron.id}) + return 0 + if not matches or cron.last_scheduled_for == occurrence: + return 0 + if cron.last_job is not None: + last_job = await uow.jobs.get(cron.last_job) + if last_job is not None and last_job.status in { + JobStatus.PENDING, + JobStatus.RUNNING, + }: + return 0 + try: + job = await JobManager.create_in_uow( + cron.job_type, cron.job_parameters, cron.job_timeout_seconds, uow=uow + ) + except Exception as error: + raise InvalidCronTemplateError(f"Invalid Cron {cron.id} template") from error + cron.last_job = job.id + cron.last_scheduled_for = occurrence + await uow.crons.save(cron) + emit_event("job.submitted", {"inkcre.job.id": job.id or 0, "inkcre.cron.id": cron_id}) + return 1 + + @classmethod + async def run_now(cls, cron_id: CronID) -> JobModel: + """Create a Job from one Cron template without changing Cron progress.""" + async with cron_uow() as uow: + cron = await uow.crons.get(cron_id) + if cron is None: + raise CronNotFoundError(f"Cron {cron_id} does not exist") + job = await JobManager.create_in_uow( + cron.job_type, + cron.job_parameters, + cron.job_timeout_seconds, + uow=uow, + ) + emit_event("job.submitted", {"inkcre.job.id": job.id or 0, "inkcre.cron.id": cron_id}) + return job + + @classmethod + async def create(cls, form: CronForm) -> CronModel: + """Validate and create one Cron template.""" + if len(form.schedule.split()) != 5 or not croniter.croniter.is_valid(form.schedule): + raise InvalidCronScheduleError( + "Cron schedule must be a valid five-field UNIX expression" + ) + async with cron_uow() as uow: + cron = CronModel(**form.model_dump()) + with input_path("job_parameters"): + cron.job_parameters = await JobManager.normalize_parameters( + form.job_type, form.job_parameters, uow + ) + await uow.crons.save(cron) + return cron + + @classmethod + async def update(cls, cron_id: CronID, form: CronForm) -> CronModel: + """Validate and replace the editable fields of one Cron template.""" + if len(form.schedule.split()) != 5 or not croniter.croniter.is_valid(form.schedule): + raise InvalidCronScheduleError( + "Cron schedule must be a valid five-field UNIX expression" + ) + async with cron_uow() as uow: + cron = await uow.crons.get(cron_id) + if cron is None: + raise CronNotFoundError(f"Cron {cron_id} does not exist") + cron.schedule = form.schedule + cron.enabled = form.enabled + cron.job_type = form.job_type + with input_path("job_parameters"): + cron.job_parameters = await JobManager.normalize_parameters( + form.job_type, form.job_parameters, uow + ) + cron.job_timeout_seconds = form.job_timeout_seconds + await uow.crons.save(cron) + return cron + + @classmethod + async def get(cls, cron_id: CronID) -> CronModel | None: + async with cron_uow() as uow: + return await uow.crons.get(cron_id) + + @classmethod + async def list_crons( + cls, *, limit: int | None = None, cursor: CronID | None = None + ) -> tuple[list[CronModel], CronID | None]: + async with cron_uow() as uow: + return await uow.crons.list(limit=limit, cursor=cursor) + + @classmethod + async def patch(cls, cron_id: CronID, form: CronUpdateForm) -> CronModel: + async with cron_uow() as uow: + cron = await uow.crons.get(cron_id, lock=True) + if cron is None: + raise CronNotFoundError(f"Cron {cron_id} does not exist") + changes = form.model_dump(exclude_unset=True) + candidate = CronForm.model_validate( + { + **{field: getattr(cron, field) for field in CronForm.model_fields}, + **changes, + } + ) + if len(candidate.schedule.split()) != 5 or not croniter.croniter.is_valid( + candidate.schedule + ): + raise InvalidCronScheduleError( + "Cron schedule must be a valid five-field UNIX expression" + ) + with input_path("job_parameters"): + parameters = await JobManager.normalize_parameters( + candidate.job_type, candidate.job_parameters, uow + ) + for field in changes: + setattr(cron, field, getattr(candidate, field)) + cron.job_parameters = parameters + await uow.crons.save(cron) + return cron + + @classmethod + async def delete(cls, cron_id: CronID) -> bool: + async with cron_uow() as uow: + cron = await uow.crons.get(cron_id) + if cron is None: + return False + await uow.crons.delete(cron) + return True diff --git a/inkcre_core/business/deployment_config.py b/inkcre_core/business/deployment_config.py new file mode 100644 index 00000000..cf50bd24 --- /dev/null +++ b/inkcre_core/business/deployment_config.py @@ -0,0 +1,186 @@ +"""Deployment-scoped configuration registry and persistence.""" + +import typing +import dataclasses + +import pydantic + +from inkcre_core.persistence.deployment_config.uow import configuration_transaction +from inkcre_core.configuration import ConfigContract +from inkcre_core.schemas.deployment_config import ( + DeploymentConfigKey, + DeploymentConfigModel, + DeploymentConfigSchemaID, + DeploymentConfigView, +) + + +class UnknownDeploymentConfigSchemaError(LookupError): + """The current peer has no validator for an exact persisted schema ID.""" + + +class DeploymentConfigSchemaCollisionError(ValueError): + """Two different local models claimed the same exact schema ID.""" + + +class DeploymentConfigNotFoundError(LookupError): + """A patch addressed a deployment config that does not exist.""" + + +@dataclasses.dataclass(frozen=True) +class _SchemaRegistration: + contract: ConfigContract + keys: tuple[str, ...] + + +class DeploymentConfigManager: + """Own exact schema registration and the shared ``configs`` relation.""" + + _contracts: dict[DeploymentConfigSchemaID, _SchemaRegistration] = {} + + @classmethod + def register_schema( + cls, + schema_id: DeploymentConfigSchemaID, + model: type[pydantic.BaseModel], + *, + keys: tuple[str, ...] = (), + ) -> None: + """Register one exact schema ID idempotently for the same model.""" + existing = cls._contracts.get(schema_id) + if existing is not None: + if existing.contract.model is model: + cls._contracts[schema_id] = _SchemaRegistration( + existing.contract, tuple(sorted(set(existing.keys) | set(keys))) + ) + return + raise DeploymentConfigSchemaCollisionError( + f"Deployment config schema {schema_id!r} is already registered " + f"by {existing.contract.model.__qualname__}" + ) + cls._contracts[schema_id] = _SchemaRegistration(ConfigContract(model), keys) + + @classmethod + def _contract( + cls, + schema_id: DeploymentConfigSchemaID, + ) -> ConfigContract: + try: + return cls._contracts[schema_id].contract + except KeyError as error: + raise UnknownDeploymentConfigSchemaError( + f"Unknown deployment config schema: {schema_id}" + ) from error + + @classmethod + def _restore_record( + cls, + record: DeploymentConfigModel, + ) -> pydantic.BaseModel: + return cls._contract(record.schema_id).validate(record.value) + + @classmethod + def _view(cls, record: DeploymentConfigModel) -> DeploymentConfigView: + return DeploymentConfigView( + key=record.key, + schema=record.schema_id, + value=record.value, + created_at=record.created_at, + updated_at=record.updated_at, + ) + + @classmethod + def get_schema(cls, schema_id: str, *, include_schema: bool = True) -> dict: + contract = cls._contract(schema_id) + entry = cls._contracts[schema_id] + result: dict[str, typing.Any] = { + "id": schema_id, + "keys": entry.keys, + "description": contract.model.__doc__ or "", + } + if include_schema: + result["input_schema"] = contract.json_schema() + return result + + @classmethod + def list_schemas( + cls, *, limit: int | None = None, cursor: str | None = None + ) -> tuple[list[dict], str | None]: + ids = sorted(key for key in cls._contracts if cursor is None or key > cursor) + more = limit is not None and len(ids) > limit + ids = ids[:limit] + return [cls.get_schema(key, include_schema=False) for key in ids], ids[ + -1 + ] if more else None + + +class DeploymentConfigService: + """Deployment configuration use cases over the process-local schema registry.""" + + @staticmethod + async def get(key: DeploymentConfigKey) -> pydantic.BaseModel | None: + async with configuration_transaction() as configs: + record = await configs.get(key) + return None if record is None else DeploymentConfigManager._restore_record(record) + + @staticmethod + async def read(key: DeploymentConfigKey) -> DeploymentConfigView | None: + async with configuration_transaction() as configs: + record = await configs.get(key) + return None if record is None else DeploymentConfigManager._view(record) + + @staticmethod + async def replace( + key: DeploymentConfigKey, + schema_id: DeploymentConfigSchemaID, + complete_value: dict[str, typing.Any], + ) -> DeploymentConfigView: + result, _ = await DeploymentConfigService.replace_with_status( + key, schema_id, complete_value + ) + return result + + @staticmethod + async def replace_with_status( + key: DeploymentConfigKey, + schema_id: DeploymentConfigSchemaID, + complete_value: dict[str, typing.Any], + ) -> tuple[DeploymentConfigView, bool]: + normalized = DeploymentConfigManager._contract(schema_id).normalize(complete_value) + async with configuration_transaction() as configs: + record, created = await configs.replace(key, schema_id, normalized) + result = DeploymentConfigManager._view(record) + return result, created + + @staticmethod + async def list_configs( + *, limit: int | None = None, cursor: str | None = None + ) -> tuple[list[DeploymentConfigView], str | None]: + async with configuration_transaction() as configs: + rows, next_cursor = await configs.list(limit=limit, cursor=cursor) + return [DeploymentConfigManager._view(row) for row in rows], next_cursor + + @staticmethod + async def delete(key: DeploymentConfigKey) -> bool: + async with configuration_transaction() as configs: + record = await configs.get(key) + if record is None: + return False + await configs.delete(record) + return True + + @staticmethod + async def patch( + key: DeploymentConfigKey, partial_value: dict[str, typing.Any] + ) -> DeploymentConfigView: + async with configuration_transaction() as configs: + record = await configs.get(key, for_update=True) + if record is None: + raise DeploymentConfigNotFoundError(f"Deployment config {key!r} does not exist") + contract = DeploymentConfigManager._contract(record.schema_id) + record.value = contract.prepare_patch(record.value, partial_value).model_dump( + mode="json" + ) + await configs.save(record) + result = DeploymentConfigManager._view(record) + return result diff --git a/inkcre_core/business/graph_navigation_retrieval/__init__.py b/inkcre_core/business/graph_navigation_retrieval/__init__.py new file mode 100644 index 00000000..0b10b714 --- /dev/null +++ b/inkcre_core/business/graph_navigation_retrieval/__init__.py @@ -0,0 +1,3 @@ +from .main import GraphNavigationRetrievalManager + +__all__ = ["GraphNavigationRetrievalManager"] diff --git a/inkcre_core/business/graph_navigation_retrieval/main.py b/inkcre_core/business/graph_navigation_retrieval/main.py new file mode 100644 index 00000000..1da4c943 --- /dev/null +++ b/inkcre_core/business/graph_navigation_retrieval/main.py @@ -0,0 +1,476 @@ +"""Bounded, presentation-neutral navigation over persisted graph authority.""" + +from collections import deque +import typing + + +from inkcre_core.persistence.info_base.uow import GraphUnitOfWork, graph_uow +from inkcre_core.schemas.graph_navigation_retrieval import ( + DEFAULT_NEIGHBORHOOD_LIMIT, + MAX_NEIGHBORHOOD_LIMIT, + DEFAULT_MAX_HOPS, + MAX_MAX_HOPS, + DEFAULT_MAX_EXPLORED_BLOCKS, + MAX_MAX_EXPLORED_BLOCKS, + DEFAULT_MAX_EXPLORED_RELATIONS, + BlockNeighborhood, + ConnectedComponentsResult, + ConnectedSeedComponent, + GraphDirection, + GraphModel, + PathFound, + PathLimitReached, + PathNotFound, + PathResult, + RelationNeighborhood, +) +from inkcre_core.schemas.info_base.block import BlockID, BlockModel +from inkcre_core.schemas.info_base.relation import RelationID, RelationModel + + +FRONTIER_QUERY_SIZE = 200 + + +class GraphNavigationRetrievalManager: + """Own bounded graph-navigation queries over persisted entities.""" + + @classmethod + async def get_random_block( + cls, + ) -> BlockModel | None: + async with graph_uow() as uow: + return await uow.blocks.get_random() + + @classmethod + async def get_block_neighborhood( + cls, + focal_block: BlockID, + *, + direction: GraphDirection = "both", + contents: typing.Collection[str] = (), + limit: int = DEFAULT_NEIGHBORHOOD_LIMIT, + cursor: RelationID | None = None, + ) -> BlockNeighborhood | None: + if not 1 <= limit <= MAX_NEIGHBORHOOD_LIMIT: + raise ValueError(f"limit must be between 1 and {MAX_NEIGHBORHOOD_LIMIT}") + async with graph_uow() as uow: + focal = await uow.blocks.get(focal_block) + if focal is None: + return None + + requested = limit + 1 + branches: list[tuple[RelationModel, ...]] = [] + if direction in ("out", "both"): + branches.append( + await uow.relations.get_endpoint_page( + (focal_block,), + endpoint="from", + contents=contents, + cursor=cursor, + limit=requested, + ) + ) + if direction in ("in", "both"): + branches.append( + await uow.relations.get_endpoint_page( + (focal_block,), endpoint="to", contents=contents, cursor=cursor, limit=requested + ) + ) + by_id = { + relation.id: relation + for branch in branches + for relation in branch + if relation.id is not None + } + ordered = sorted( + by_id.values(), + key=lambda relation: typing.cast(int, relation.id), + reverse=True, + ) + page = ordered[:requested] + has_more = len(page) > limit + page = page[:limit] + endpoint_ids = {focal_block} + for relation in page: + endpoint_ids.update((relation.from_, relation.to_)) + blocks = await uow.blocks.get_many(endpoint_ids) + existing_ids = {block.id for block in blocks} + closed_relations = tuple( + relation + for relation in page + if relation.from_ in existing_ids and relation.to_ in existing_ids + ) + next_cursor = typing.cast(RelationID, page[-1].id) if has_more and page else None + return BlockNeighborhood( + focal_block=focal_block, + graph=GraphModel(blocks=blocks, relations=closed_relations), + next_cursor=next_cursor, + ) + + @classmethod + async def get_relation_neighborhood( + cls, + focal_relation: RelationID, + ) -> RelationNeighborhood | None: + async with graph_uow() as uow: + relation = await uow.relations.get_by_id(focal_relation) + if relation is None: + return None + blocks = await uow.blocks.get_many((relation.from_, relation.to_)) + if {block.id for block in blocks} != {relation.from_, relation.to_}: + return None + return RelationNeighborhood( + focal_relation=focal_relation, + graph=GraphModel(blocks=blocks, relations=(relation,)), + ) + + @classmethod + async def get_connected_components( + cls, + seed_block_ids: typing.Collection[BlockID], + *, + contents: typing.Collection[str], + max_explored_blocks: int = DEFAULT_MAX_EXPLORED_BLOCKS, + max_explored_relations: int = DEFAULT_MAX_EXPLORED_RELATIONS, + ) -> ConnectedComponentsResult: + """Partition existing seeds by bounded undirected exact-content reachability.""" + seeds = tuple(dict.fromkeys(seed_block_ids)) + relation_contents = tuple(dict.fromkeys(contents)) + if not relation_contents: + raise ValueError("contents must not be empty") + if max_explored_blocks < 1 or max_explored_relations < 1: + raise ValueError("exploration bounds must be positive") + if len(seeds) > max_explored_blocks: + raise ValueError("seed blocks exceed max_explored_blocks") + async with graph_uow() as uow: + existing_blocks = await uow.blocks.get_many(seeds) + existing_seed_ids = {block.id for block in existing_blocks if block.id is not None} + missing = tuple(seed for seed in seeds if seed not in existing_seed_ids) + assigned_seeds: set[BlockID] = set() + explored_blocks = set(existing_seed_ids) + seen_relations: set[RelationID] = set() + explored_relation_count = 0 + proof_relations: dict[RelationID, RelationModel] = {} + components: list[ConnectedSeedComponent] = [] + truncated = False + + for seed in seeds: + if seed not in existing_seed_ids or seed in assigned_seeds: + continue + if truncated: + components.append( + ConnectedSeedComponent(seed_block_ids=(seed,), member_block_ids=(seed,)) + ) + assigned_seeds.add(seed) + continue + + members = {seed} + frontier = deque((seed,)) + while frontier and not truncated: + current = frontier.popleft() + for endpoint in ("from", "to"): + cursor: RelationID | None = None + while not truncated: + remaining = max_explored_relations - explored_relation_count + if remaining == 0: + truncated = True + break + requested = min(FRONTIER_QUERY_SIZE, remaining + 1) + page = await uow.relations.get_endpoint_page( + (current,), + endpoint=typing.cast(typing.Literal["from", "to"], endpoint), + contents=relation_contents, + cursor=cursor, + limit=requested, + ) + if len(page) > remaining: + page = page[:remaining] + truncated = True + explored_relation_count += len(page) + for relation in page: + if relation.id is None or relation.id in seen_relations: + continue + relation_id = relation.id + seen_relations.add(relation_id) + neighbor = relation.to_ if relation.from_ == current else relation.from_ + if neighbor in members: + continue + if ( + neighbor not in explored_blocks + and len(explored_blocks) >= max_explored_blocks + ): + truncated = True + break + members.add(neighbor) + explored_blocks.add(neighbor) + frontier.append(neighbor) + proof_relations[relation_id] = relation + if truncated or len(page) < requested: + break + cursor = typing.cast(RelationID, page[-1].id) + + component_seeds = tuple(seed_id for seed_id in seeds if seed_id in members) + assigned_seeds.update(component_seeds) + components.append( + ConnectedSeedComponent( + seed_block_ids=component_seeds, + member_block_ids=tuple(sorted(members)), + ) + ) + + proof_blocks = await uow.blocks.get_many(explored_blocks) + return ConnectedComponentsResult( + components=tuple(components), + proof_graph=GraphModel( + blocks=proof_blocks, + relations=tuple(proof_relations.values()), + ), + missing_seed_block_ids=missing, + truncated=truncated, + ) + + @classmethod + async def find_path( # noqa: PLR0913 + cls, + from_block: BlockID, + to_block: BlockID, + *, + direction: GraphDirection = "both", + contents: typing.Collection[str] = (), + max_hops: int = DEFAULT_MAX_HOPS, + max_explored_blocks: int = DEFAULT_MAX_EXPLORED_BLOCKS, + ) -> PathResult: + if not 0 <= max_hops <= MAX_MAX_HOPS: + raise ValueError(f"max_hops must be between 0 and {MAX_MAX_HOPS}") + if not 1 <= max_explored_blocks <= MAX_MAX_EXPLORED_BLOCKS: + raise ValueError( + f"max_explored_blocks must be between 1 and {MAX_MAX_EXPLORED_BLOCKS}" + ) + async with graph_uow() as uow: + endpoints = await uow.blocks.get_many((from_block, to_block)) + if {block.id for block in endpoints} != {from_block, to_block}: + return PathNotFound() + if from_block == to_block: + return PathFound( + graph=GraphModel(blocks=endpoints, relations=()), + block_path=(from_block,), + relation_path=(), + ) + + forward_parents: dict[BlockID, tuple[BlockID, RelationID] | None] = {from_block: None} + backward_next: dict[BlockID, tuple[BlockID, RelationID] | None] = {to_block: None} + forward_depths = {from_block: 0} + backward_depths = {to_block: 0} + forward_frontier = {from_block} + backward_frontier = {to_block} + traversed_relations: dict[RelationID, RelationModel] = {} + + while forward_frontier and backward_frontier: + forward_level = forward_depths[next(iter(forward_frontier))] + backward_level = backward_depths[next(iter(backward_frontier))] + if forward_level + backward_level >= max_hops: + return PathLimitReached() + + expand_forward = len(forward_frontier) <= len(backward_frontier) + if expand_forward: + next_frontier, meeting = await cls._expand_path_frontier( + forward_frontier, + direction=direction, + contents=contents, + reverse=False, + own_steps=forward_parents, + own_depths=forward_depths, + other_depths=backward_depths, + max_hops=max_hops, + traversed_relations=traversed_relations, + uow=uow, + ) + forward_frontier = next_frontier + else: + next_frontier, meeting = await cls._expand_path_frontier( + backward_frontier, + direction=direction, + contents=contents, + reverse=True, + own_steps=backward_next, + own_depths=backward_depths, + other_depths=forward_depths, + max_hops=max_hops, + traversed_relations=traversed_relations, + uow=uow, + ) + backward_frontier = next_frontier + explored = set(forward_parents) | set(backward_next) + if len(explored) > max_explored_blocks: + return PathLimitReached() + if meeting is not None: + return await cls._assemble_bidirectional_path( + from_block, + to_block, + meeting=meeting, + forward_parents=forward_parents, + backward_next=backward_next, + traversed_relations=traversed_relations, + uow=uow, + ) + return PathNotFound() + + @classmethod + async def _expand_path_frontier( # noqa: PLR0913 + cls, + frontier: set[BlockID], + *, + direction: GraphDirection, + contents: typing.Collection[str], + reverse: bool, + own_steps: dict[BlockID, tuple[BlockID, RelationID] | None], + own_depths: dict[BlockID, int], + other_depths: dict[BlockID, int], + max_hops: int, + traversed_relations: dict[RelationID, RelationModel], + uow: GraphUnitOfWork, + ) -> tuple[set[BlockID], BlockID | None]: + next_frontier: set[BlockID] = set() + meeting: BlockID | None = None + best_hops: int | None = None + frontier_items = tuple(frontier) + for chunk_start in range(0, len(frontier), FRONTIER_QUERY_SIZE): + chunk = frontier_items[chunk_start : chunk_start + FRONTIER_QUERY_SIZE] + chunk_members = set(chunk) + relations = await cls._frontier_relations( + chunk, + direction=direction, + contents=contents, + reverse=reverse, + uow=uow, + ) + for relation in relations: + relation_id = typing.cast(RelationID, relation.id) + traversed_relations[relation_id] = relation + for current, neighbor in cls._relation_steps( + relation, frontier=chunk_members, direction=direction, reverse=reverse + ): + if neighbor in own_steps: + continue + own_steps[neighbor] = (current, relation_id) + own_depths[neighbor] = own_depths[current] + 1 + next_frontier.add(neighbor) + if neighbor not in other_depths: + continue + hops = own_depths[neighbor] + other_depths[neighbor] + if hops <= max_hops and (best_hops is None or hops < best_hops): + best_hops = hops + meeting = neighbor + return next_frontier, meeting + + @staticmethod + def _relation_steps( + relation: RelationModel, + *, + frontier: set[BlockID], + direction: GraphDirection, + reverse: bool, + ) -> tuple[tuple[BlockID, BlockID], ...]: + steps: list[tuple[BlockID, BlockID]] = [] + if not reverse: + if direction in ("out", "both") and relation.from_ in frontier: + steps.append((relation.from_, relation.to_)) + if direction in ("in", "both") and relation.to_ in frontier: + steps.append((relation.to_, relation.from_)) + else: + if direction in ("out", "both") and relation.to_ in frontier: + steps.append((relation.to_, relation.from_)) + if direction in ("in", "both") and relation.from_ in frontier: + steps.append((relation.from_, relation.to_)) + return tuple(steps) + + @classmethod + async def _frontier_relations( + cls, + frontier: typing.Collection[BlockID], + *, + direction: GraphDirection, + contents: typing.Collection[str], + reverse: bool = False, + uow: GraphUnitOfWork, + ) -> tuple[RelationModel, ...]: + relations: dict[RelationID, RelationModel] = {} + needs_from = ( + direction in ("out", "both") + if not reverse + else direction + in ( + "in", + "both", + ) + ) + needs_to = ( + direction in ("in", "both") + if not reverse + else direction + in ( + "out", + "both", + ) + ) + if needs_from: + for relation in await uow.relations.get_endpoint_page( + frontier, endpoint="from", contents=contents, limit=MAX_MAX_EXPLORED_BLOCKS + ): + if relation.id is not None: + relations[relation.id] = relation + if needs_to: + for relation in await uow.relations.get_endpoint_page( + frontier, endpoint="to", contents=contents, limit=MAX_MAX_EXPLORED_BLOCKS + ): + if relation.id is not None: + relations[relation.id] = relation + return tuple(relations.values()) + + @classmethod + async def _assemble_bidirectional_path( # noqa: PLR0913 + cls, + from_block: BlockID, + to_block: BlockID, + *, + meeting: BlockID, + forward_parents: dict[BlockID, tuple[BlockID, RelationID] | None], + backward_next: dict[BlockID, tuple[BlockID, RelationID] | None], + traversed_relations: dict[RelationID, RelationModel], + uow: GraphUnitOfWork, + ) -> PathResult: + block_path: list[BlockID] = [meeting] + relation_path: list[RelationID] = [] + current = meeting + while current != from_block: + parent = forward_parents[current] + if parent is None: # pragma: no cover - reconstruction invariant + raise RuntimeError("Path reconstruction lost its parent") + current, relation_id = parent + block_path.append(current) + relation_path.append(relation_id) + block_path.reverse() + relation_path.reverse() + current = meeting + while current != to_block: + next_step = backward_next[current] + if next_step is None: # pragma: no cover - reconstruction invariant + raise RuntimeError("Path reconstruction lost its next step") + current, relation_id = next_step + block_path.append(current) + relation_path.append(relation_id) + blocks = await uow.blocks.get_many(block_path) + relations = tuple( + relation + for relation_id in relation_path + if (relation := traversed_relations.get(relation_id)) is not None + ) + if {block.id for block in blocks} != set(block_path) or len(relations) != len( + relation_path + ): + return PathNotFound() + return PathFound( + graph=GraphModel(blocks=blocks, relations=relations), + block_path=tuple(block_path), + relation_path=tuple(relation_path), + ) diff --git a/inkcre_core/business/graph_navigation_retrieval/tools.py b/inkcre_core/business/graph_navigation_retrieval/tools.py new file mode 100644 index 00000000..837bab78 --- /dev/null +++ b/inkcre_core/business/graph_navigation_retrieval/tools.py @@ -0,0 +1,148 @@ +"""Agent Tool controllers for graph-navigation retrieval.""" + +import typing + +import pydantic + +from inkcre_core.business.agent import AgentManager, ToolExecutionError +from inkcre_core.business.agent.projection import project_json +from inkcre_core.schemas.ai import JSONValue +from inkcre_core.schemas.graph_navigation_retrieval import ( + DEFAULT_MAX_EXPLORED_BLOCKS, + DEFAULT_MAX_EXPLORED_RELATIONS, + DEFAULT_MAX_HOPS, + DEFAULT_NEIGHBORHOOD_LIMIT, + MAX_MAX_EXPLORED_BLOCKS, + MAX_MAX_HOPS, + MAX_NEIGHBORHOOD_LIMIT, + GraphDirection, +) +from inkcre_core.schemas.info_base.block import BlockID +from inkcre_core.schemas.info_base.relation import RelationID + +from .main import GraphNavigationRetrievalManager + + +GET_ENTITY_NEIGHBORHOOD_TOOL = "get_entity_neighborhood" +FIND_PATH_TOOL = "find_path" +GET_CONNECTED_COMPONENTS_TOOL = "get_connected_components" + + +class BlockNeighborhoodInput(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + entity_type: typing.Literal["block"] + entity_id: BlockID + direction: GraphDirection = "both" + contents: tuple[str, ...] = () + limit: int = pydantic.Field( + default=DEFAULT_NEIGHBORHOOD_LIMIT, ge=1, le=MAX_NEIGHBORHOOD_LIMIT + ) + cursor: RelationID | None = None + + +class RelationNeighborhoodInput(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + entity_type: typing.Literal["relation"] + entity_id: RelationID + + +class EntityNeighborhoodInput( + pydantic.RootModel[ + typing.Annotated[ + BlockNeighborhoodInput | RelationNeighborhoodInput, + pydantic.Field(discriminator="entity_type"), + ] + ] +): + model_config = pydantic.ConfigDict(json_schema_extra={"type": "object"}) + + @classmethod + def model_json_schema(cls, *args, **kwargs) -> dict[str, typing.Any]: + schema = super().model_json_schema(*args, **kwargs) + properties = BlockNeighborhoodInput.model_json_schema(*args, **kwargs)["properties"] + properties["entity_type"] = {"type": "string", "enum": ["block", "relation"]} + schema["properties"] = properties + return schema + + +class FindPathInput(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + from_block_id: BlockID + to_block_id: BlockID + direction: GraphDirection = "both" + contents: tuple[str, ...] = () + max_hops: int = pydantic.Field(default=DEFAULT_MAX_HOPS, ge=0, le=MAX_MAX_HOPS) + max_explored_blocks: int = pydantic.Field( + default=DEFAULT_MAX_EXPLORED_BLOCKS, ge=1, le=MAX_MAX_EXPLORED_BLOCKS + ) + + +class ConnectedComponentsInput(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + seed_block_ids: tuple[BlockID, ...] + contents: tuple[str, ...] = pydantic.Field(min_length=1) + max_explored_blocks: int = pydantic.Field(default=DEFAULT_MAX_EXPLORED_BLOCKS, ge=1) + max_explored_relations: int = pydantic.Field(default=DEFAULT_MAX_EXPLORED_RELATIONS, ge=1) + + +@AgentManager.tool( + GET_ENTITY_NEIGHBORHOOD_TOOL, + description=( + "Read a Block's direct neighborhood or a Relation with its endpoints. " + "Null may indicate an incorrect entity type." + ), +) +async def get_entity_neighborhood(input: EntityNeighborhoodInput) -> JSONValue: + request = input.root + if request.entity_type == "block": + result = await GraphNavigationRetrievalManager.get_block_neighborhood( + request.entity_id, + direction=request.direction, + contents=request.contents, + limit=request.limit, + cursor=request.cursor, + ) + else: + result = await GraphNavigationRetrievalManager.get_relation_neighborhood( + request.entity_id + ) + return project_json(result) + + +@AgentManager.tool( + FIND_PATH_TOOL, + description="Find a bounded graph path; an exploration limit is not proof of absence.", +) +async def find_path(input: FindPathInput) -> JSONValue: + return project_json( + await GraphNavigationRetrievalManager.find_path( + input.from_block_id, + input.to_block_id, + direction=input.direction, + contents=input.contents, + max_hops=input.max_hops, + max_explored_blocks=input.max_explored_blocks, + ) + ) + + +@AgentManager.tool( + GET_CONNECTED_COMPONENTS_TOOL, + description=( + "Partition seeds by bounded undirected reachability through exact Relation contents." + ), +) +async def get_connected_components(input: ConnectedComponentsInput) -> JSONValue: + try: + result = await GraphNavigationRetrievalManager.get_connected_components( + **input.model_dump() + ) + except ValueError as error: + raise ToolExecutionError( + {"error": type(error).__name__, "message": str(error)} + ) from error + return project_json(result) diff --git a/app/business/info_base/AGENTS.md b/inkcre_core/business/info_base/AGENTS.md similarity index 92% rename from app/business/info_base/AGENTS.md rename to inkcre_core/business/info_base/AGENTS.md index 7245ba4d..9d2f722a 100644 --- a/app/business/info_base/AGENTS.md +++ b/inkcre_core/business/info_base/AGENTS.md @@ -1,6 +1,6 @@ # info_base/ Local Guide -本文件只描述 `app/business/info_base/` 的局部事实、术语和编辑边界。跨 +本文件只描述 `inkcre_core/business/info_base/` 的局部事实、术语和编辑边界。跨 `extension/source/info_base/application` 的结构先读 [business-pipeline-and-authority.md](../../../docs/30-unit-tdd/business-pipeline-and-authority.md)。 @@ -18,7 +18,7 @@ - `relation.py`：relation CRUD/fetchsert、direct direction query 与 endpoint-label dynamic-property projection； - `resolver/`：exact decoder registry 与 use projection； - `storage/`：opaque pointer → actual bytes； -- `app/schemas/info_base/block.py`：persisted block 与 instance-local hydration cache。 +- `inkcre_core/schemas/info_base/block.py`：persisted block 与 instance-local hydration cache。 ## Content Vocabulary @@ -90,6 +90,6 @@ ## 异步数据库边界 - 新 Graph 应用入口位于 `commands.py`，事务内组合使用必填 `GraphUnitOfWork`，不能传 raw session。 -- `app/persistence/info_base/repository.py` 只接收 session 并执行查询／写入／flush，不允许创建 session 或 commit/rollback。 -- `app/persistence/info_base/uow.py` 拥有工厂组合，使用 SQLAlchemy 原生 framing，不跨 task 共享。 +- `inkcre_core/persistence/info_base/repository.py` 只接收 session 并执行查询／写入／flush，不允许创建 session 或 commit/rollback。 +- `inkcre_core/persistence/info_base/uow.py` 拥有工厂组合，使用 SQLAlchemy 原生 framing，不跨 task 共享。 - 独立入口使用 BlockService／RelationService；组合写入调用 persist_graph／persist_stars。不保留同步或可选 session 接口。 diff --git a/inkcre_core/business/info_base/__init__.py b/inkcre_core/business/info_base/__init__.py new file mode 100644 index 00000000..72c49b13 --- /dev/null +++ b/inkcre_core/business/info_base/__init__.py @@ -0,0 +1,14 @@ +from .main import InfoBaseManager +from .commands import persist_graph, submit_graph, persist_stars, submit_stars +from .services import BlockService, RelationService, get_entity_records + +__all__ = [ + "InfoBaseManager", + "persist_graph", + "submit_graph", + "persist_stars", + "submit_stars", + "get_entity_records", + "BlockService", + "RelationService", +] diff --git a/inkcre_core/business/info_base/commands.py b/inkcre_core/business/info_base/commands.py new file mode 100644 index 00000000..f53e6e91 --- /dev/null +++ b/inkcre_core/business/info_base/commands.py @@ -0,0 +1,79 @@ +"""Graph application operations shared by transports and composing use cases.""" + +from inkcre_core.schemas.info_base.main import ( + GraphBlockIDMapping, + GraphForm, + SubmitGraphResult, + StarsGraphForm, +) +from inkcre_core.schemas.info_base.block import BlockModel +from inkcre_core.schemas.info_base.relation import RelationModel +from inkcre_core.schemas.info_base.relation import RelationCreateForm + +from inkcre_core.persistence.info_base.uow import GraphUnitOfWork, graph_uow + + +async def persist_graph(graph: GraphForm, uow: GraphUnitOfWork) -> SubmitGraphResult: + """Add a flat graph to the supplied unit of work without ending its transaction.""" + blocks = await uow.blocks.create_many(graph.blocks) + mappings: list[GraphBlockIDMapping] = [] + for form, block in zip(graph.blocks, blocks, strict=True): + if block.id is None: + raise RuntimeError("Inserted Block is missing its database ID") + mappings.append(GraphBlockIDMapping(local_id=form.id, id=block.id)) + persisted_ids = {mapping.local_id: mapping.id for mapping in mappings} + await uow.relations.create_many( + RelationCreateForm( + from_=persisted_ids.get(form.from_, form.from_), + to_=persisted_ids.get(form.to_, form.to_), + content=form.content, + ) + for form in graph.relations + ) + return SubmitGraphResult(blocks=tuple(mappings)) + + +async def submit_graph(graph: GraphForm) -> SubmitGraphResult: + """Commit one complete flat graph before returning its Block identity mapping.""" + async with graph_uow() as uow: + result = await persist_graph(graph, uow) + return result + + +async def get_related_block( + block_id: int, *, content: str, outgoing: bool = True +) -> BlockModel | None: + async with graph_uow() as uow: + return await uow.blocks.get_related(block_id, content=content, outgoing=outgoing) + + +async def persist_stars(graph: StarsGraphForm, uow: GraphUnitOfWork) -> BlockModel: + """Reconcile exact resolver identities and relations within one transaction.""" + from .resolver import ResolverManager + + candidate = BlockModel.model_validate(graph.block) + existing = await ResolverManager.get(candidate).get_existing_async(uow.blocks) + block = existing if existing is not None else await uow.blocks.create(graph.block) + if block.id is None: + raise RuntimeError("Persisted star Block is missing its database ID") + for arc in graph.out_arcs: + target = await persist_stars(arc.to_graph, uow) + if target.id is None: + raise RuntimeError("Persisted star Block is missing its database ID") + await uow.relations.fetchsert( + RelationModel(from_=block.id, to_=target.id, content=arc.relation.content) + ) + for arc in graph.in_arcs: + source = await persist_stars(arc.from_graph, uow) + if source.id is None: + raise RuntimeError("Persisted star Block is missing its database ID") + await uow.relations.fetchsert( + RelationModel(from_=source.id, to_=block.id, content=arc.relation.content) + ) + return block + + +async def submit_stars(graph: StarsGraphForm) -> BlockModel: + async with graph_uow() as uow: + result = await persist_stars(graph, uow) + return result diff --git a/inkcre_core/business/info_base/main.py b/inkcre_core/business/info_base/main.py new file mode 100644 index 00000000..b4099c6d --- /dev/null +++ b/inkcre_core/business/info_base/main.py @@ -0,0 +1,123 @@ +"""Info-base graph command coordination and cross-retrieval use cases.""" + +from __future__ import annotations + +import asyncio +from dataclasses import dataclass +import typing + +from inkcre_core.business.info_base.services import BlockService, get_entity_records +from inkcre_core.schemas.lexical_retrieval import LexicalRetrievalResult +from inkcre_core.schemas.semantic_retrieval import ( + SemanticRetrievalResult, + VectorRetrievalOptions, +) +from inkcre_core.schemas.info_base.block import BlockModel +from inkcre_core.schemas.info_base.relation import RelationModel + +from inkcre_core.schemas.info_base.main import ( + GraphBlockForm, + GraphForm, + GraphRelationForm, + StarsGraphForm, +) + + +@dataclass(frozen=True) +class RetrievalBranch: + result: LexicalRetrievalResult | SemanticRetrievalResult | None = None + error: BaseException | None = None + + +class InfoBaseManager: + """Own graph commands and use cases that span info-base read capabilities.""" + + @classmethod + async def retrieve( + cls, + query: str, + mode: typing.Literal["lexical", "semantic", "hybrid"] = "hybrid", + limit: int = 20, + ) -> dict[str, RetrievalBranch]: + """Run selected retrieval owners independently without merging their ranking.""" + from inkcre_core.business.lexical_retrieval import LexicalRetrievalManager + from inkcre_core.business.semantic_retrieval import SemanticRetrievalManager + + operations: dict[str, typing.Awaitable] = {} + if mode in {"lexical", "hybrid"}: + operations["lexical"] = LexicalRetrievalManager.retrieve_local(query, limit) + if mode in {"semantic", "hybrid"}: + operations["semantic"] = SemanticRetrievalManager.retrieve_local( + query, options=VectorRetrievalOptions(limit=limit) + ) + outcomes = await asyncio.gather(*operations.values(), return_exceptions=True) + return { + name: RetrievalBranch( + error=outcome if isinstance(outcome, BaseException) else None, + result=None if isinstance(outcome, BaseException) else outcome, + ) + for name, outcome in zip(operations, outcomes, strict=True) + } + + @classmethod + async def get_entities( + cls, + entities: typing.Collection[tuple[typing.Literal["block", "relation"], int]], + *, + random_count: int = 1, + ) -> tuple[BlockModel | RelationModel | None, ...]: + """Read ordered persisted entities, or random Blocks when selection is empty.""" + if not entities: + return typing.cast( + tuple[BlockModel | RelationModel | None, ...], + await BlockService.get_random_many(random_count), + ) + blocks, relations = await get_entity_records( + tuple(identity for kind, identity in entities if kind == "block"), + tuple(identity for kind, identity in entities if kind == "relation"), + ) + return tuple( + blocks.get(identity) if kind == "block" else relations.get(identity) + for kind, identity in entities + ) + + @classmethod + def normalize_graph( + cls, + stars: StarsGraphForm, + id_start: int = -1, + ) -> GraphForm: + """Flatten recursive authoring while allocating deterministic negative IDs.""" + if id_start >= 0: + raise ValueError("id_start must be negative") + + next_id = id_start + blocks: list[GraphBlockForm] = [] + relations: list[GraphRelationForm] = [] + + def visit(star: StarsGraphForm) -> int: + nonlocal next_id + block_id = next_id + next_id -= 1 + blocks.append(GraphBlockForm.model_validate(star.block, update={"id": block_id})) + + for arc in star.out_arcs: + to_id = visit(arc.to_graph) + relations.append( + GraphRelationForm.model_validate( + arc.relation, + update={"from_": block_id, "to_": to_id}, + ) + ) + for arc in star.in_arcs: + from_id = visit(arc.from_graph) + relations.append( + GraphRelationForm.model_validate( + arc.relation, + update={"from_": from_id, "to_": block_id}, + ) + ) + return block_id + + visit(stars) + return GraphForm(blocks=tuple(blocks), relations=tuple(relations)) diff --git a/app/business/info_base/resolver/AGENTS.md b/inkcre_core/business/info_base/resolver/AGENTS.md similarity index 97% rename from app/business/info_base/resolver/AGENTS.md rename to inkcre_core/business/info_base/resolver/AGENTS.md index 690a6e17..339e99c4 100644 --- a/app/business/info_base/resolver/AGENTS.md +++ b/inkcre_core/business/info_base/resolver/AGENTS.md @@ -1,6 +1,6 @@ # resolver/ Local Guide -本文件只描述 `app/business/info_base/resolver/` 的局部事实与高风险边界。上层 contract 看 +本文件只描述 `inkcre_core/business/info_base/resolver/` 的局部事实与高风险边界。上层 contract 看 [info_base guide](../AGENTS.md) 与 [business-pipeline-and-authority.md](../../../../docs/30-unit-tdd/business-pipeline-and-authority.md)。 diff --git a/inkcre_core/business/info_base/resolver/__init__.py b/inkcre_core/business/info_base/resolver/__init__.py new file mode 100644 index 00000000..edf45ec6 --- /dev/null +++ b/inkcre_core/business/info_base/resolver/__init__.py @@ -0,0 +1,60 @@ +from .bootstrap import register_core_resolvers +from .contracts import ( + CORE_RESOLVER_IDS, + CoreResolverID, + DuplicateResolverRegistrationError, + ResolverContentError, + ResolverContractError, + TextProjectionContext, + UnknownDraftResolverError, + UnknownResolverError, + UnsupportedResolverCapability, + ResolverMethodInputError, + UnknownResolverMethodError, +) +from .main import ( + Resolver, + ResolverDraftCapability, + ResolverManager, + ResolverMethodContract, +) + +__all__ = [ + "ResolverManager", + "Resolver", + "register_core_resolvers", + "CORE_RESOLVER_IDS", + "CoreResolverID", + "ResolverContractError", + "TextProjectionContext", + "UnknownResolverError", + "UnknownDraftResolverError", + "DuplicateResolverRegistrationError", + "ResolverContentError", + "UnsupportedResolverCapability", + "ResolverDraftCapability", + "ResolverMethodContract", + "ResolverMethodInputError", + "UnknownResolverMethodError", + "AudioResolver", + "EPUBResolver", + "FileResolver", + "ImageResolver", + "PDFResolver", + "VideoResolver", + "TextResolver", + "MarkdownResolver", + "HTMLResolver", + "ZIPResolver", +] + +from .audio import AudioResolver +from .epub import EPUBResolver +from .file import FileResolver +from .image import ImageResolver +from .pdf import PDFResolver +from .video import VideoResolver +from .text import TextResolver +from .markdown import MarkdownResolver +from .html import HTMLResolver +from .zip import ZIPResolver diff --git a/inkcre_core/business/info_base/resolver/audio.py b/inkcre_core/business/info_base/resolver/audio.py new file mode 100644 index 00000000..76efc17e --- /dev/null +++ b/inkcre_core/business/info_base/resolver/audio.py @@ -0,0 +1,173 @@ +"""Exact audio resolver using metadata-only PyAV inspection.""" + +import asyncio +from dataclasses import dataclass +from io import BytesIO +import typing + +import av +from av.stream import Disposition +import pydantic + +from inkcre_core.business.deployment_config import ( + DeploymentConfigManager, + DeploymentConfigService, +) +from inkcre_core.schemas.ai import AudioContentPart + +from .contracts import ( + ResolverContentError, + TextProjectionContext, + UnsupportedResolverCapability, +) +from .inspection import ( + ByteContentFacts, + detect_media_type, + format_lexical_facts, + require_bytes, +) +from .main import Resolver +from .materialization import try_materialize_model_text + + +AUDIO_RESOLVER_CONFIG_KEY = "core.resolver.audio" +AUDIO_RESOLVER_CONFIG_SCHEMA = "core.resolver.audio.config.v1" + + +class AudioResolverConfig(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + transcript_model: int + + +DeploymentConfigManager.register_schema( + AUDIO_RESOLVER_CONFIG_SCHEMA, AudioResolverConfig, keys=(AUDIO_RESOLVER_CONFIG_KEY,) +) + + +@dataclass(frozen=True, slots=True) +class AudioSolvedContent(ByteContentFacts): + container: str | None + codec: str | None + duration_ms: int | None + channels: int | None + sample_rate_hz: int | None + bitrate_bps: int | None + + +def _container_name(format_name: str | None) -> str | None: + if not format_name: + return None + names = format_name.split(",") + return "mp4" if "mp4" in names else names[0] + + +def _duration_ms(container, stream) -> int | None: + if stream.duration is not None and stream.time_base is not None: + return round(float(stream.duration * stream.time_base) * 1000) + if container.duration is not None: + return round(container.duration / 1000) + return None + + +def _primary_stream(streams): + default = [stream for stream in streams if bool(stream.disposition & Disposition.default)] + ordinary = [ + stream for stream in streams if not bool(stream.disposition & Disposition.attached_pic) + ] + candidates = default or ordinary or list(streams) + return candidates[0] if candidates else None + + +def _inspect_audio(content: bytes) -> AudioSolvedContent: + try: + with av.open(BytesIO(content), mode="r") as container: + stream = _primary_stream(container.streams.audio) + if stream is None: + raise ResolverContentError("core.audio.v1", "no audio stream") + codec = stream.codec_context + facts = AudioSolvedContent( + content=content, + byte_size=len(content), + detected_media_type=detect_media_type(content), + container=_container_name(container.format.name), + codec=codec.name, + duration_ms=_duration_ms(container, stream), + channels=codec.channels, + sample_rate_hz=codec.sample_rate, + bitrate_bps=stream.bit_rate or container.bit_rate, + ) + except ResolverContentError: + raise + except (av.FFmpegError, OSError, ValueError) as error: + raise ResolverContentError("core.audio.v1", "invalid audio") from error + return facts + + +class AudioResolver( + Resolver[AudioSolvedContent, bytes], + rso_type="core.audio.v1", +): + async def _get_solved_content( + self, + *, + refresh: bool = False, + materialize_missing: bool = True, + ) -> AudioSolvedContent: + del materialize_missing + content = require_bytes( + await self.get_raw_content(refresh=refresh), + self.__rsotype__, + ) + return await asyncio.to_thread(_inspect_audio, content) + + async def get_text( + self, + *, + context: TextProjectionContext = "default", + refresh: bool = False, + materialize_missing: bool = True, + ) -> str: + if context == "default": + raise UnsupportedResolverCapability(self.__rsotype__, "text") + solved = await self.get_solved_content(refresh=refresh, materialize_missing=False) + if materialize_missing: + config = typing.cast( + AudioResolverConfig | None, + await DeploymentConfigService.get(AUDIO_RESOLVER_CONFIG_KEY), + ) + if config is not None: + media_type = solved.detected_media_type or ( + f"audio/{solved.container}" if solved.container is not None else None + ) + if media_type is not None: + await try_materialize_model_text( + block_id=self.block_id, + role="transcript", + model=config.transcript_model, + instruction=( + "Transcribe only speech and authored spoken language in the audio. " + "Return plain text with no summary or explanation." + ), + media=AudioContentPart( + data=solved.content, + mime_type=media_type, + transfer_url=await self.get_transfer_url(), + ), + ) + return format_lexical_facts( + "audio", + ( + ("media type", solved.detected_media_type), + ("container", solved.container), + ("codec", solved.codec), + ("duration ms", solved.duration_ms), + ("channels", solved.channels), + ("sample rate Hz", solved.sample_rate_hz), + ("bitrate bps", solved.bitrate_bps), + ), + ) + + async def get_label(self, *, refresh: bool = False) -> str: + del refresh + return "audio" diff --git a/inkcre_core/business/info_base/resolver/bootstrap.py b/inkcre_core/business/info_base/resolver/bootstrap.py new file mode 100644 index 00000000..862a2379 --- /dev/null +++ b/inkcre_core/business/info_base/resolver/bootstrap.py @@ -0,0 +1,31 @@ +"""Explicit registration of resolver decoders owned by impl-py.""" + +from .main import ResolverManager + + +def register_core_resolvers() -> None: + """Register built-in decoders independently of extension loading.""" + from .audio import AudioResolver + from .epub import EPUBResolver + from .file import FileResolver + from .html import HTMLResolver + from .image import ImageResolver + from .pdf import PDFResolver + from .text import TextResolver + from .markdown import MarkdownResolver + from .video import VideoResolver + from .zip import ZIPResolver + + for resolver_class in ( + TextResolver, + MarkdownResolver, + HTMLResolver, + ImageResolver, + AudioResolver, + VideoResolver, + PDFResolver, + EPUBResolver, + ZIPResolver, + FileResolver, + ): + ResolverManager.register_resolver(resolver_class) diff --git a/inkcre_core/business/info_base/resolver/contracts.py b/inkcre_core/business/info_base/resolver/contracts.py new file mode 100644 index 00000000..fbcfa89b --- /dev/null +++ b/inkcre_core/business/info_base/resolver/contracts.py @@ -0,0 +1,95 @@ +"""Stable resolver IDs, projection controls, and typed contract failures.""" + +from typing import Literal, TypeAlias + +import pydantic + + +TextProjectionContext: TypeAlias = Literal["default", "lexical"] + + +CoreResolverID: TypeAlias = Literal[ + "core.text.v1", + "core.markdown.v1", + "core.html.v1", + "core.image.v1", + "core.audio.v1", + "core.video.v1", + "core.pdf.v1", + "core.epub.v1", + "core.zip.v1", + "core.file.v1", +] + +CORE_RESOLVER_IDS: tuple[CoreResolverID, ...] = ( + "core.text.v1", + "core.markdown.v1", + "core.html.v1", + "core.image.v1", + "core.audio.v1", + "core.video.v1", + "core.pdf.v1", + "core.epub.v1", + "core.zip.v1", + "core.file.v1", +) + + +class ResolverContractError(RuntimeError): + """Base failure for resolver selection and capability contracts.""" + + +class ResolverMethodInputError(pydantic.ValidationError): + """Native input details, distinguishable from errors inside a content method.""" + + +class UnknownResolverMethodError(ResolverContractError, ValueError): + """An exact Resolver has no projected read method with this name.""" + + +class UnknownResolverError(ResolverContractError): + """No exact resolver decoder is registered for a persisted resolver ID.""" + + def __init__(self, resolver_id: str): + self.resolver_id = resolver_id + super().__init__(f"Unknown resolver ID: {resolver_id}") + + +class UnknownDraftResolverError(ResolverContractError): + """The selected exact Resolver has no graph-drafting capability.""" + + def __init__(self, resolver_id: str): + self.resolver_id = resolver_id + super().__init__(f"Resolver is not available for graph drafting: {resolver_id}") + + +class DuplicateResolverRegistrationError(ResolverContractError): + """Two different decoder classes claimed the same exact resolver ID.""" + + def __init__(self, resolver_id: str, existing: type, attempted: type): + self.resolver_id = resolver_id + self.existing = existing + self.attempted = attempted + super().__init__( + f"Resolver ID {resolver_id} is already registered by " + f"{existing.__module__}.{existing.__qualname__}; " + f"cannot register {attempted.__module__}.{attempted.__qualname__}" + ) + + +class UnsupportedResolverCapability(ResolverContractError): + """The exact resolver contract does not provide one requested projection.""" + + def __init__(self, resolver_id: str, capability: str): + self.resolver_id = resolver_id + self.capability = capability + super().__init__(f"Resolver {resolver_id} does not support {capability}") + + +class ResolverContentError(ResolverContractError, ValueError): + """Hydrated content does not satisfy its exact resolver contract.""" + + def __init__(self, resolver_id: str, reason: str): + self.resolver_id = resolver_id + self.reason = reason + super().__init__(f"Resolver {resolver_id} rejected content: {reason}") diff --git a/inkcre_core/business/info_base/resolver/epub.py b/inkcre_core/business/info_base/resolver/epub.py new file mode 100644 index 00000000..a403579e --- /dev/null +++ b/inkcre_core/business/info_base/resolver/epub.py @@ -0,0 +1,219 @@ +"""Exact EPUB resolver using bounded package-document inspection.""" + +import asyncio +import datetime +from dataclasses import dataclass +from io import BytesIO +import zipfile + +from lxml import etree + +from .contracts import ( + ResolverContentError, + TextProjectionContext, + UnsupportedResolverCapability, +) +from .inspection import ByteContentFacts, format_lexical_facts, require_bytes +from .label import format_label +from .main import Resolver + + +_MAX_EPUB_MEMBERS = 10_000 +_MAX_PACKAGE_BYTES = 2 * 1024 * 1024 +_XML_PARSER = etree.XMLParser( + resolve_entities=False, + no_network=True, + recover=False, + huge_tree=False, +) + + +@dataclass(frozen=True, slots=True) +class EPUBSolvedContent(ByteContentFacts): + epub_version: str | None + title: str | None + creators: tuple[str, ...] + languages: tuple[str, ...] + modified_at: datetime.datetime | None + manifest_count: int + spine_count: int + has_navigation: bool + + +def _local_name(element: etree._Element) -> str: + return etree.QName(element).localname + + +def _read_bounded( + archive: zipfile.ZipFile, + member_name: str, + *, + limit: int = _MAX_PACKAGE_BYTES, +) -> bytes: + try: + member = archive.getinfo(member_name) + except KeyError as error: + raise ResolverContentError( + "core.epub.v1", + f"missing required EPUB member {member_name}", + ) from error + if member.file_size > limit: + raise ResolverContentError( + "core.epub.v1", + f"EPUB member {member_name} exceeds inspection limit", + ) + with archive.open(member) as stream: + result = stream.read(limit + 1) + if len(result) > limit: + raise ResolverContentError( + "core.epub.v1", + f"EPUB member {member_name} exceeds inspection limit", + ) + return result + + +def _parse_xml(content: bytes, label: str) -> etree._Element: + try: + return etree.fromstring(content, parser=_XML_PARSER) + except etree.XMLSyntaxError as error: + raise ResolverContentError("core.epub.v1", f"invalid {label}") from error + + +def _first_text(elements: list[etree._Element]) -> str | None: + for element in elements: + if element.text and (value := element.text.strip()): + return value + return None + + +def _all_text(elements: list[etree._Element]) -> tuple[str, ...]: + return tuple( + value for element in elements if element.text and (value := element.text.strip()) + ) + + +def _parse_modified(value: str | None) -> datetime.datetime | None: + if value is None: + return None + try: + parsed = datetime.datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError as error: + raise ResolverContentError( + "core.epub.v1", + "invalid dcterms:modified timestamp", + ) from error + return parsed + + +def _inspect_epub(content: bytes) -> EPUBSolvedContent: + try: + with zipfile.ZipFile(BytesIO(content)) as archive: + if len(archive.infolist()) > _MAX_EPUB_MEMBERS: + raise ResolverContentError("core.epub.v1", "too many EPUB members") + if _read_bounded(archive, "mimetype", limit=128) != b"application/epub+zip": + raise ResolverContentError("core.epub.v1", "invalid EPUB mimetype member") + + container = _parse_xml( + _read_bounded(archive, "META-INF/container.xml"), + "EPUB container document", + ) + rootfiles = [ + element for element in container.iter() if _local_name(element) == "rootfile" + ] + package_path = next( + (element.get("full-path") for element in rootfiles if element.get("full-path")), + None, + ) + if package_path is None: + raise ResolverContentError("core.epub.v1", "missing EPUB package path") + package = _parse_xml( + _read_bounded(archive, package_path), + "EPUB package document", + ) + except ResolverContentError: + raise + except (zipfile.BadZipFile, OSError, ValueError) as error: + raise ResolverContentError("core.epub.v1", "invalid EPUB container") from error + + elements = list(package.iter()) + titles = [element for element in elements if _local_name(element) == "title"] + creators = [element for element in elements if _local_name(element) == "creator"] + languages = [element for element in elements if _local_name(element) == "language"] + manifest_items = [element for element in elements if _local_name(element) == "item"] + spine_items = [element for element in elements if _local_name(element) == "itemref"] + modified = next( + ( + element.text.strip() + for element in elements + if _local_name(element) == "meta" + and element.get("property") == "dcterms:modified" + and element.text + and element.text.strip() + ), + None, + ) + + return EPUBSolvedContent( + content=content, + byte_size=len(content), + detected_media_type="application/epub+zip", + epub_version=package.get("version"), + title=_first_text(titles), + creators=_all_text(creators), + languages=_all_text(languages), + modified_at=_parse_modified(modified), + manifest_count=len(manifest_items), + spine_count=len(spine_items), + has_navigation=any( + "nav" in (element.get("properties") or "").split() for element in manifest_items + ), + ) + + +class EPUBResolver( + Resolver[EPUBSolvedContent, bytes], + rso_type="core.epub.v1", +): + async def _get_solved_content( + self, + *, + refresh: bool = False, + materialize_missing: bool = True, + ) -> EPUBSolvedContent: + del materialize_missing + content = require_bytes( + await self.get_raw_content(refresh=refresh), + self.__rsotype__, + ) + return await asyncio.to_thread(_inspect_epub, content) + + async def get_text( + self, + *, + context: TextProjectionContext = "default", + refresh: bool = False, + materialize_missing: bool = True, + ) -> str: + if context == "default": + raise UnsupportedResolverCapability(self.__rsotype__, "text") + del materialize_missing + solved = await self.get_solved_content(refresh=refresh, materialize_missing=False) + return format_lexical_facts( + "EPUB", + ( + ("media type", solved.detected_media_type), + ("title", solved.title), + ("creators", ", ".join(solved.creators) or None), + ("languages", ", ".join(solved.languages) or None), + ("EPUB version", solved.epub_version), + ("manifest items", solved.manifest_count), + ("spine items", solved.spine_count), + ), + ) + + async def get_label(self, *, refresh: bool = False) -> str: + solved = await self.get_solved_content( + refresh=refresh, + materialize_missing=False, + ) + return format_label("EPUB", solved.title) diff --git a/inkcre_core/business/info_base/resolver/file.py b/inkcre_core/business/info_base/resolver/file.py new file mode 100644 index 00000000..b95167f7 --- /dev/null +++ b/inkcre_core/business/info_base/resolver/file.py @@ -0,0 +1,64 @@ +"""Exact generic-file resolver for bytes without stronger semantics.""" + +import asyncio +from dataclasses import dataclass + +from .contracts import TextProjectionContext, UnsupportedResolverCapability +from .inspection import ( + ByteContentFacts, + detect_media_type, + format_lexical_facts, + require_bytes, +) +from .main import Resolver + + +@dataclass(frozen=True, slots=True) +class FileSolvedContent(ByteContentFacts): + pass + + +def _inspect_file(content: bytes) -> FileSolvedContent: + return FileSolvedContent( + content=content, + byte_size=len(content), + detected_media_type=detect_media_type(content), + ) + + +class FileResolver( + Resolver[FileSolvedContent, bytes], + rso_type="core.file.v1", +): + async def _get_solved_content( + self, + *, + refresh: bool = False, + materialize_missing: bool = True, + ) -> FileSolvedContent: + del materialize_missing + content = require_bytes( + await self.get_raw_content(refresh=refresh), + self.__rsotype__, + ) + return await asyncio.to_thread(_inspect_file, content) + + async def get_text( + self, + *, + context: TextProjectionContext = "default", + refresh: bool = False, + materialize_missing: bool = True, + ) -> str: + if context == "default": + raise UnsupportedResolverCapability(self.__rsotype__, "text") + del materialize_missing + solved = await self.get_solved_content(refresh=refresh, materialize_missing=False) + return format_lexical_facts( + "file", + (("media type", solved.detected_media_type), ("bytes", solved.byte_size)), + ) + + async def get_label(self, *, refresh: bool = False) -> str: + del refresh + return "file" diff --git a/inkcre_core/business/info_base/resolver/html.py b/inkcre_core/business/info_base/resolver/html.py new file mode 100644 index 00000000..d5b8ba9f --- /dev/null +++ b/inkcre_core/business/info_base/resolver/html.py @@ -0,0 +1,73 @@ +"""Exact HTML-source resolver with a derived text projection.""" + +import asyncio + +import html2text +from lxml import html as lxml_html + +from .inspection import decode_html_bytes +from .label import format_label +from .main import Resolver, TextProjectionContext + + +class HTMLResolver(Resolver[str, str | bytes], rso_type="core.html.v1"): + @classmethod + def create_graph(cls, url: str): + from inkcre_core.schemas.info_base.block import BlockForm + from inkcre_core.schemas.info_base.main import StarsGraphForm + + return StarsGraphForm( + block=BlockForm(resolver=cls.__rsotype__, content=url, storage=-1) + ) + + async def _get_solved_content( + self, + *, + refresh: bool = False, + materialize_missing: bool = True, + ) -> str: + del materialize_missing + content = await self.get_raw_content(refresh=refresh) + return ( + content if isinstance(content, str) else decode_html_bytes(content, self.__rsotype__) + ) + + async def get_text( + self, + *, + context: TextProjectionContext = "default", + refresh: bool = False, + materialize_missing: bool = True, + ) -> str: + del context + source = await self.get_solved_content( + refresh=refresh, + materialize_missing=materialize_missing, + ) + if any(f"<{tag}" in source.lower() for tag in ("video", "audio", "track")): + document = lxml_html.document_fromstring(source) + for resource in document.xpath("//video | //audio | //source | //track"): + for attribute in ("src", "poster"): + if address := resource.get(attribute): + link = lxml_html.Element("a", href=address) + link.text = resource.get("label") or address + resource.addnext(link) + source = lxml_html.tostring(document, encoding="unicode") + return await asyncio.to_thread(html2text.HTML2Text().handle, source) + + async def get_label(self, *, refresh: bool = False) -> str: + source = await self.get_solved_content( + refresh=refresh, + materialize_missing=False, + ) + identifier = None + try: + document = lxml_html.fromstring(source) + values = document.xpath("//title/text()") or document.xpath( + "//h1//text() | //h2//text() | //h3//text() | //h4//text() | " + "//h5//text() | //h6//text()" + ) + identifier = next((str(value) for value in values if str(value).strip()), None) + except (TypeError, ValueError): + pass + return format_label("html", identifier) diff --git a/inkcre_core/business/info_base/resolver/image.py b/inkcre_core/business/info_base/resolver/image.py new file mode 100644 index 00000000..ffde4a74 --- /dev/null +++ b/inkcre_core/business/info_base/resolver/image.py @@ -0,0 +1,158 @@ +"""Exact image resolver using bounded Pillow header inspection.""" + +import asyncio +from dataclasses import dataclass +from io import BytesIO +import typing + +from PIL import Image, UnidentifiedImageError +import pydantic + +from inkcre_core.business.deployment_config import ( + DeploymentConfigManager, + DeploymentConfigService, +) +from inkcre_core.schemas.ai import ImageContentPart +from inkcre_core.schemas.info_base.block import BlockForm +from inkcre_core.schemas.info_base.main import OutArcForm, StarsGraphForm +from inkcre_core.schemas.info_base.relation import RelationForm + +from .contracts import ( + ResolverContentError, + TextProjectionContext, + UnsupportedResolverCapability, +) +from .inspection import ( + ByteContentFacts, + detect_media_type, + format_lexical_facts, + require_bytes, +) +from .main import Resolver +from .materialization import try_materialize_model_text + + +IMAGE_RESOLVER_CONFIG_KEY = "core.resolver.image" +IMAGE_RESOLVER_CONFIG_SCHEMA = "core.resolver.image.config.v1" + + +class ImageResolverConfig(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + text_model: int + + +DeploymentConfigManager.register_schema( + IMAGE_RESOLVER_CONFIG_SCHEMA, ImageResolverConfig, keys=(IMAGE_RESOLVER_CONFIG_KEY,) +) + + +@dataclass(frozen=True, slots=True) +class ImageSolvedContent(ByteContentFacts): + format: str | None + width: int | None + height: int | None + frame_count: int | None + + +def _inspect_image(content: bytes) -> ImageSolvedContent: + try: + with Image.open(BytesIO(content)) as image: + image_format = image.format.lower() if image.format else None + width, height = image.size + frame_count = getattr(image, "n_frames", None) + image.verify() + except (UnidentifiedImageError, OSError, SyntaxError, ValueError) as error: + raise ResolverContentError("core.image.v1", "invalid image") from error + return ImageSolvedContent( + content=content, + byte_size=len(content), + detected_media_type=detect_media_type(content), + format=image_format, + width=width, + height=height, + frame_count=frame_count, + ) + + +class ImageResolver( + Resolver[ImageSolvedContent, bytes], + rso_type="core.image.v1", +): + @classmethod + def create_graph(cls, url: str, alt_text: str | None = None) -> StarsGraphForm: + from .text import TextResolver + + return StarsGraphForm( + block=BlockForm(resolver=cls.__rsotype__, content=url, storage=-1), + out_arcs=( + OutArcForm( + relation=RelationForm(content="alt:text"), + to_graph=TextResolver.create_graph(alt_text), + ), + ) + if alt_text + else (), + ) + + async def _get_solved_content( + self, + *, + refresh: bool = False, + materialize_missing: bool = True, + ) -> ImageSolvedContent: + del materialize_missing + content = require_bytes( + await self.get_raw_content(refresh=refresh), + self.__rsotype__, + ) + return await asyncio.to_thread(_inspect_image, content) + + async def get_text( + self, + *, + context: TextProjectionContext = "default", + refresh: bool = False, + materialize_missing: bool = True, + ) -> str: + if context == "default": + raise UnsupportedResolverCapability(self.__rsotype__, "text") + solved = await self.get_solved_content(refresh=refresh, materialize_missing=False) + if materialize_missing: + config = typing.cast( + ImageResolverConfig | None, + await DeploymentConfigService.get(IMAGE_RESOLVER_CONFIG_KEY), + ) + if config is not None: + media_type = solved.detected_media_type or ( + f"image/{solved.format}" if solved.format is not None else None + ) + if media_type is not None: + await try_materialize_model_text( + block_id=self.block_id, + role="text", + model=config.text_model, + instruction=( + "Transcribe only written text visibly present in the image. " + "Return plain text with no explanation; return no content when none exists." + ), + media=ImageContentPart( + data=solved.content, + mime_type=media_type, + transfer_url=await self.get_transfer_url(), + ), + ) + return format_lexical_facts( + "image", + ( + ("media type", solved.detected_media_type), + ("format", solved.format), + ("width", solved.width), + ("height", solved.height), + ("frames", solved.frame_count), + ), + ) + + async def get_label(self, *, refresh: bool = False) -> str: + del refresh + return "image" diff --git a/inkcre_core/business/info_base/resolver/inspection.py b/inkcre_core/business/info_base/resolver/inspection.py new file mode 100644 index 00000000..4ac77f4f --- /dev/null +++ b/inkcre_core/business/info_base/resolver/inspection.py @@ -0,0 +1,104 @@ +"""Bounded implementation helpers shared by exact core content resolvers.""" + +from dataclasses import dataclass +from collections.abc import Iterable +import codecs +import re + +import puremagic + +from .contracts import ResolverContentError + + +@dataclass(frozen=True, slots=True) +class ByteContentFacts: + """Peer-local actual bytes plus common byte-derived facts.""" + + content: bytes + byte_size: int + detected_media_type: str | None + + +def detect_media_type(content: bytes) -> str | None: + """Return bounded signature detection without inventing an unknown value.""" + try: + detected = puremagic.from_string(content, mime=True) + except puremagic.PureError: + return None + return detected if isinstance(detected, str) else None + + +def decode_unicode_bytes(content: bytes, resolver_id: str) -> str: + """Decode a Unicode BOM when present, otherwise require strict UTF-8.""" + decoders = ( + (codecs.BOM_UTF32_LE, "utf-32"), + (codecs.BOM_UTF32_BE, "utf-32"), + (codecs.BOM_UTF8, "utf-8-sig"), + (codecs.BOM_UTF16_LE, "utf-16"), + (codecs.BOM_UTF16_BE, "utf-16"), + ) + encoding = next( + (candidate for bom, candidate in decoders if content.startswith(bom)), + "utf-8", + ) + try: + return content.decode(encoding, errors="strict") + except UnicodeError as error: + raise ResolverContentError(resolver_id, f"invalid {encoding} text") from error + + +_HTML_CHARSET_SEARCH_BYTES = 4096 +_HTML_CHARSET_PATTERNS = ( + re.compile(rb"]*charset\s*=\s*['\"]?\s*([a-zA-Z0-9._-]+)", re.I), + re.compile(rb"]*content\s*=\s*['\"][^'\"]*charset=([a-zA-Z0-9._-]+)", re.I), +) + + +def decode_html_bytes(content: bytes, resolver_id: str) -> str: + """Decode HTML using a Unicode BOM, bounded meta declaration, or strict UTF-8.""" + for bom in ( + codecs.BOM_UTF32_LE, + codecs.BOM_UTF32_BE, + codecs.BOM_UTF8, + codecs.BOM_UTF16_LE, + codecs.BOM_UTF16_BE, + ): + if content.startswith(bom): + return decode_unicode_bytes(content, resolver_id) + + prefix = content[:_HTML_CHARSET_SEARCH_BYTES] + declared_encoding = next( + ( + match.group(1).decode("ascii") + for pattern in _HTML_CHARSET_PATTERNS + if (match := pattern.search(prefix)) is not None + ), + "utf-8", + ) + try: + encoding = codecs.lookup(declared_encoding).name + except LookupError as error: + raise ResolverContentError( + resolver_id, + f"unknown declared HTML charset {declared_encoding}", + ) from error + try: + return content.decode(encoding, errors="strict") + except UnicodeError as error: + raise ResolverContentError(resolver_id, f"invalid {encoding} HTML") from error + + +def require_bytes(content: object, resolver_id: str) -> bytes: + if not isinstance(content, bytes): + raise ResolverContentError(resolver_id, "byte-oriented content must be storage-backed") + return content + + +def format_lexical_facts( + kind: str, + facts: Iterable[tuple[str, object | None]], +) -> str: + """Format bounded Resolver-owned metadata without serializing arbitrary objects.""" + lines = [kind] + lines.extend(f"{label}: {value}" for label, value in facts if value is not None) + return "\n".join(lines) diff --git a/inkcre_core/business/info_base/resolver/label.py b/inkcre_core/business/info_base/resolver/label.py new file mode 100644 index 00000000..a271cb2d --- /dev/null +++ b/inkcre_core/business/info_base/resolver/label.py @@ -0,0 +1,31 @@ +"""Shared mechanics for concise Resolver-owned Block labels.""" + +LABEL_IDENTIFIER_LIMIT = 96 + + +def normalize_label_identifier( + value: str | None, + *, + first_line: bool = False, +) -> str | None: + """Normalize whitespace and bound one optional human-readable identifier.""" + if value is None: + return None + candidates = value.splitlines() if first_line else (value,) + normalized = next((" ".join(item.split()) for item in candidates if item.strip()), "") + if not normalized: + return None + if len(normalized) > LABEL_IDENTIFIER_LIMIT: + return f"{normalized[:LABEL_IDENTIFIER_LIMIT]}…" + return normalized + + +def format_label( + kind: str, + identifier: str | None = None, + *, + first_line: bool = False, +) -> str: + """Format one stable resolver-qualified label without exposing exact IDs.""" + normalized = normalize_label_identifier(identifier, first_line=first_line) + return kind if normalized is None else f"{kind} <{normalized}>" diff --git a/inkcre_core/business/info_base/resolver/main.py b/inkcre_core/business/info_base/resolver/main.py new file mode 100644 index 00000000..c3a59cc2 --- /dev/null +++ b/inkcre_core/business/info_base/resolver/main.py @@ -0,0 +1,434 @@ +import abc +from collections.abc import Collection +from dataclasses import dataclass +import inspect +import typing +from typing import Optional as Opt + +import pydantic + +from inkcre_core.business.info_base.services import RelationService +from inkcre_core.schemas.info_base.main import StarsGraphForm +from inkcre_core.schemas.info_base.block import BlockForm, BlockID, ResolverType, BlockModel +from inkcre_core.schemas.info_base.relation import RelationModel +from inkcre_core.schemas.info_base.storage import StorageID + +from inkcre_core.persistence.info_base.repository import BlockRepository + +from .contracts import ( + DuplicateResolverRegistrationError, + TextProjectionContext, + UnknownDraftResolverError, + UnknownResolverError, + UnknownResolverMethodError, + ResolverMethodInputError, +) + + +@dataclass(frozen=True) +class ResolverDraftCapability: + """One code-owned Resolver graph-drafting contract.""" + + resolver: ResolverType + description: str + input_model: type[pydantic.BaseModel] + resolver_cls: type["Resolver"] + + +@dataclass(frozen=True) +class ResolverMethodContract: + """One typed public read method on a registered Resolver.""" + + name: str + description: str + input_model: type[pydantic.BaseModel] + + @property + def input_schema(self) -> dict[str, typing.Any]: + return self.input_model.model_json_schema() + + +class ResolverManager: + RESOLVER_CLS: dict[ResolverType, type["Resolver"]] = {} + """Global resolver registry. + + Map ResolverType to Resolver class + """ + + @classmethod + def register_resolver(cls, resolver_cls: type["Resolver"]) -> None: + existing = cls.RESOLVER_CLS.get(resolver_cls.__rsotype__) + if existing is resolver_cls: + return + if existing is not None: + raise DuplicateResolverRegistrationError( + resolver_cls.__rsotype__, + existing, + resolver_cls, + ) + cls.RESOLVER_CLS[resolver_cls.__rsotype__] = resolver_cls + + @classmethod + def get(cls, block: BlockModel) -> "Resolver": + """Create resolver instance from block.""" + try: + resolver_cls = cls.RESOLVER_CLS[block.resolver] + except KeyError as error: + raise UnknownResolverError(block.resolver) from error + return resolver_cls(block) + + @classmethod + def get_draft_capabilities(cls) -> tuple[ResolverDraftCapability, ...]: + """Snapshot currently registered Resolvers that can draft rooted graphs.""" + capabilities = ( + ResolverDraftCapability( + resolver=resolver_id, + description=resolver_cls.draft_description, + input_model=resolver_cls.draft_input_model, + resolver_cls=resolver_cls, + ) + for resolver_id, resolver_cls in cls.RESOLVER_CLS.items() + if resolver_cls.draft_description is not None + and resolver_cls.draft_input_model is not None + ) + return tuple(sorted(capabilities, key=lambda capability: capability.resolver)) + + @classmethod + def get_draft_capability( + cls, + resolver: ResolverType, + ) -> ResolverDraftCapability: + """Select one exact graph-drafting contract without fallback.""" + for capability in cls.get_draft_capabilities(): + if capability.resolver == resolver: + return capability + raise UnknownDraftResolverError(resolver) + + @classmethod + def get_method_contracts( + cls, + resolver: ResolverType, + ) -> tuple[ResolverMethodContract, ...]: + """Discover typed public read methods on one registered Resolver.""" + resolver_cls = cls.RESOLVER_CLS.get(resolver) + if resolver_cls is None: + return () + return cls._method_contracts(resolver_cls) + + @classmethod + def get_common_method_contracts(cls) -> tuple[ResolverMethodContract, ...]: + """Common Resolver reads, available without per-type discovery.""" + return cls._method_contracts(Resolver) + + @staticmethod + def _method_contracts( + resolver_cls: type["Resolver"], + ) -> tuple[ResolverMethodContract, ...]: + contracts: list[ResolverMethodContract] = [] + for name, function in inspect.getmembers(resolver_cls, predicate=inspect.isfunction): + if name.startswith("_") or not name.startswith(("get_", "read_")): + continue + try: + signature = inspect.signature(function, eval_str=True) + fields: dict[str, tuple[typing.Any, typing.Any]] = {} + for parameter in signature.parameters.values(): + if parameter.name == "self": + continue + if parameter.kind in (parameter.VAR_POSITIONAL, parameter.VAR_KEYWORD): + raise TypeError("Variadic Resolver methods are not projectable") + if parameter.annotation is inspect.Parameter.empty: + raise TypeError("Resolver method parameters must be typed") + default = ( + ... if parameter.default is inspect.Parameter.empty else parameter.default + ) + annotation = parameter.annotation + if typing.get_origin(annotation) is Collection: + item_type = typing.get_args(annotation)[0] + annotation = tuple[item_type, ...] + fields[parameter.name] = (annotation, default) + input_model = typing.cast(typing.Any, pydantic.create_model)( + f"{resolver_cls.__name__}_{name}_Arguments", + __config__=pydantic.ConfigDict(extra="forbid"), + **fields, + ) + input_model.model_json_schema() + except (NameError, TypeError, pydantic.PydanticSchemaGenerationError): + continue + contracts.append( + ResolverMethodContract( + name=name, + description=inspect.getdoc(function) or name.replace("_", " "), + input_model=input_model, + ) + ) + return tuple(contracts) + + @classmethod + def get_method_contract( + cls, + resolver: ResolverType, + name: str, + ) -> ResolverMethodContract | None: + return next( + ( + contract for contract in cls.get_method_contracts(resolver) if contract.name == name + ), + None, + ) + + @classmethod + async def invoke_method( + cls, + block: BlockModel, + name: str, + arguments: dict[str, typing.Any], + ) -> typing.Any: + """Validate and invoke one projected read method on an exact Block Resolver.""" + resolver = cls.get(block) + contract = cls.get_method_contract(block.resolver, name) + if contract is None: + raise UnknownResolverMethodError( + f"Resolver {block.resolver!r} has no method {name!r}" + ) + try: + validated = contract.input_model.model_validate(arguments) + except pydantic.ValidationError as error: + raise ResolverMethodInputError.from_exception_data( + error.title, typing.cast(typing.Any, error.errors(include_url=False)) + ) from error + # Keep nested Python values typed when calling the annotated domain method; + # model_dump would convert nested input models back into dictionaries. + value = getattr(resolver, name)( + **{field: getattr(validated, field) for field in contract.input_model.model_fields} + ) + return await value if inspect.isawaitable(value) else value + + @classmethod + def match_media_type(cls, media_type: str | None) -> ResolverType | None: + """Map one specific media type to an installed exact core resolver ID. + + Evidence precedence and generic/file fallback remain extension-owned. + """ + if media_type is None: + return None + normalized = media_type.partition(";")[0].strip().lower() + if not normalized or normalized in { + "application/octet-stream", + "binary/octet-stream", + "application/binary", + }: + return None + + exact = { + "text/plain": "core.text.v1", + "text/markdown": "core.markdown.v1", + "text/html": "core.html.v1", + "application/xhtml+xml": "core.html.v1", + "application/pdf": "core.pdf.v1", + "application/epub+zip": "core.epub.v1", + "application/zip": "core.zip.v1", + "application/x-zip-compressed": "core.zip.v1", + } + resolver_id = exact.get(normalized) + if resolver_id is None: + family = normalized.partition("/")[0] + resolver_id = { + "image": "core.image.v1", + "audio": "core.audio.v1", + "video": "core.video.v1", + }.get(family) + return resolver_id if resolver_id in cls.RESOLVER_CLS else None + + +SolvedContentTV = typing.TypeVar("SolvedContentTV") +RawContentTV = typing.TypeVar("RawContentTV") +_UNSET = object() + + +class Resolver(abc.ABC, typing.Generic[SolvedContentTV, RawContentTV]): + """Resolver resolves a star graph (a block and its direct relations) + + :tparam SolvedContentTV: The type of the solved content + :tparam RawContentTV: The type of the raw content + """ + + __rsotype__: ResolverType + """Resolver type + + Extension resolvers should be namespaced and versioned, e.g., + `extensions.twitter.tweet.v1`. + """ + + draft_description: typing.ClassVar[str | None] = None + draft_input_model: typing.ClassVar[type[pydantic.BaseModel] | None] = None + + def __init_subclass__(cls, rso_type: str, **kwargs) -> None: + cls.__rsotype__ = rso_type + ResolverManager.register_resolver(cls) + return super().__init_subclass__(**kwargs) + + def __init__(self, block: BlockModel, relations: Opt[tuple[RelationModel, ...]] = None): + """Should never override __init__ in subclasses, use __post_init__ instead. + + :param block: Block to resolve. + :param relations: Relations of the block. + """ + self._block = block + self.__relations: dict[tuple[bool, bool], tuple[RelationModel, ...]] = {} + if relations is not None: + self.__relations[(True, True)] = relations + self.__solved_content: SolvedContentTV | object = _UNSET + """Solved content is the content the resolver really works with, + commonly from raw content. + """ + inline_content = ( + typing.cast(RawContentTV, self._block.content) + if self._block.storage is None + else None + ) + self.__post_init__(inline_content) + + def __post_init__(self, raw_content: Opt[RawContentTV] = None) -> None: + """Subclass post-initialization hook. + + It's suggest to set __solved_content here if possible: + ```python + async def __post_init__(self, raw_content): + if raw_content is not None: + ... # anyhow from raw_content + self.set_solved_content(solved_content) + ``` + """ + ... + + @property + def block_id(self) -> BlockID: + """Get the block ID.""" + return typing.cast(BlockID, self._block.id) + + async def get_raw_content( + self, + *, + refresh: typing.Annotated[ + bool, pydantic.Field(description="Reread current content.") + ] = False, + ) -> RawContentTV: + """Read hydrated content: text or bytes, not a storage pointer.""" + return typing.cast( + RawContentTV, + await self._block.get_hydrated_content(refresh=refresh), + ) + + async def get_transfer_url(self) -> str | None: + """Get a content transfer URL when available.""" + if self._block.storage is None: + return None + from inkcre_core.business.info_base.storage import StorageManager + + storage = await StorageManager.get_storage_async(self._block.storage) + return storage.get_transfer_url(self._block.content) + + async def get_solved_content( + self, + *, + refresh: typing.Annotated[ + bool, pydantic.Field(description="Reread current content.") + ] = False, + materialize_missing: typing.Annotated[ + bool, pydantic.Field(description="Allow creation of missing derived information.") + ] = True, + ) -> SolvedContentTV: + """Read the Resolver's typed interpretation of content.""" + if refresh or self.__solved_content is _UNSET: + self.__solved_content = await self._get_solved_content( + refresh=refresh, + materialize_missing=materialize_missing, + ) + return typing.cast(SolvedContentTV, self.__solved_content) + + async def _get_solved_content( + self, + *, + refresh: bool = False, + materialize_missing: bool = True, + ) -> SolvedContentTV: + """Get the solved content (non-cache). + + Description: + When you say "will be resolved from relations", + it means you acquire the resolver of the relation's other side block, + and use its solved content. + """ + del materialize_missing + return typing.cast( + SolvedContentTV, + await self.get_raw_content(refresh=refresh), + ) + + def set_solved_content(self, content: SolvedContentTV) -> None: + self.__solved_content = content + + async def get_relations( + self, + *, + include_in: typing.Annotated[ + bool, pydantic.Field(description="Include relations pointing to this Block.") + ] = True, + include_out: typing.Annotated[ + bool, pydantic.Field(description="Include relations pointing from this Block.") + ] = True, + refresh: bool = False, + ) -> tuple[RelationModel, ...]: + """Read direct relations of this Block.""" + key = (include_in, include_out) + if refresh or key not in self.__relations: + all_relations = None if refresh else self.__relations.get((True, True)) + if all_relations is not None: + self.__relations[key] = tuple( + relation + for relation in all_relations + if (include_in and relation.to_ == self.block_id) + or (include_out and relation.from_ == self.block_id) + ) + else: + self.__relations[key] = await RelationService.get( + block_id=self.block_id, + include_in=include_in, + include_out=include_out, + ) + return self.__relations[key] + + @classmethod + # @abc.abstractmethod TODO + def create_block(cls, content, storage: Opt[StorageID] = None) -> BlockForm: ... + + @classmethod + # @abc.abstractmethod TODO + def create_graph(cls, *args, **kwargs) -> StarsGraphForm: ... + + @abc.abstractmethod + async def get_text( + self, + *, + context: typing.Annotated[ + TextProjectionContext, + pydantic.Field(description="Lexical projection is Block-local and non-recursive."), + ] = "default", + refresh: typing.Annotated[ + bool, pydantic.Field(description="Reread current content.") + ] = False, + materialize_missing: typing.Annotated[ + bool, pydantic.Field(description="Allow creation of missing derived information.") + ] = True, + ) -> str | None: + """Read a text projection; unsupported, absent and empty are distinct.""" + ... + + @abc.abstractmethod + async def get_label(self, *, refresh: bool = False) -> str: + """Read a concise label for this Block.""" + ... + + async def get_existing_async(self, blocks: BlockRepository) -> Opt[BlockModel]: + """Reconcile the exact resolver/content identity in the caller's transaction.""" + return await blocks.find_content(self._block.resolver, self._block.content) diff --git a/inkcre_core/business/info_base/resolver/markdown.py b/inkcre_core/business/info_base/resolver/markdown.py new file mode 100644 index 00000000..e381cde1 --- /dev/null +++ b/inkcre_core/business/info_base/resolver/markdown.py @@ -0,0 +1,35 @@ +"""Markdown source remains the exact text projection; rendering is a reader concern.""" + +from .inspection import decode_unicode_bytes +from .label import format_label +from .main import Resolver, TextProjectionContext + + +class MarkdownResolver(Resolver[str, str | bytes], rso_type="core.markdown.v1"): + async def _get_solved_content( + self, *, refresh: bool = False, materialize_missing: bool = True + ) -> str: + content = await self.get_raw_content(refresh=refresh) + return ( + content + if isinstance(content, str) + else decode_unicode_bytes(content, self.__rsotype__) + ) + + async def get_text( + self, + *, + context: TextProjectionContext = "default", + refresh: bool = False, + materialize_missing: bool = True, + ) -> str: + return await self.get_solved_content( + refresh=refresh, materialize_missing=materialize_missing + ) + + async def get_label(self, *, refresh: bool = False) -> str: + return format_label( + "markdown", + await self.get_text(refresh=refresh, materialize_missing=False), + first_line=True, + ) diff --git a/inkcre_core/business/info_base/resolver/materialization.py b/inkcre_core/business/info_base/resolver/materialization.py new file mode 100644 index 00000000..2744a8e9 --- /dev/null +++ b/inkcre_core/business/info_base/resolver/materialization.py @@ -0,0 +1,80 @@ +"""Shared mechanics for Resolver-owned faithful text derivations.""" + +import logging + +from inkcre_core.business.ai import AIManager +from inkcre_core.schemas.ai import SystemMessage, UserContentPart, UserMessage + + +logger = logging.getLogger(__name__) + + +async def generate_faithful_text( + *, + model: int, + instruction: str, + media: UserContentPart, +) -> str | None: + """Ask one exact configured Model for source-faithful text, without graph access.""" + response = await AIManager.chat( + model, + ( + SystemMessage(content=instruction), + UserMessage(content=(media,)), + ), + ) + text = (response.content or "").strip() + return text or None + + +async def materialize_text_child(block_id: int, role: str, text: str) -> None: + """Best-effort idempotent graph write for one exact information role.""" + from inkcre_core.schemas.info_base.main import ( + GraphBlockForm, + GraphForm, + GraphRelationForm, + ) + + from inkcre_core.business.info_base.commands import persist_graph + from inkcre_core.persistence.info_base.uow import graph_uow + + # Recheck after provider work in the same short scope as the graph insertion. + async with graph_uow() as uow: + if await uow.blocks.get_related(block_id, content=role) is not None: + return + await persist_graph( + GraphForm( + blocks=(GraphBlockForm(id=-1, resolver="core.text.v1", content=text),), + relations=(GraphRelationForm(from_=block_id, to_=-1, content=role),), + ), + uow, + ) + + +async def try_materialize_model_text( + *, + block_id: int, + role: str, + model: int, + instruction: str, + media: UserContentPart, +) -> None: + """Keep one unavailable derivation shallow while preserving diagnostic evidence.""" + from inkcre_core.business.info_base.commands import get_related_block + + if await get_related_block(block_id, content=role) is not None: + return + try: + text = await generate_faithful_text( + model=model, + instruction=instruction, + media=media, + ) + if text is not None: + await materialize_text_child(block_id, role, text) + except Exception: + logger.warning( + "Resolver text materialization unavailable", + exc_info=True, + extra={"block": block_id, "role": role, "model": model}, + ) diff --git a/inkcre_core/business/info_base/resolver/pdf.py b/inkcre_core/business/info_base/resolver/pdf.py new file mode 100644 index 00000000..9c0ea924 --- /dev/null +++ b/inkcre_core/business/info_base/resolver/pdf.py @@ -0,0 +1,131 @@ +"""Exact PDF resolver using bounded structural inspection.""" + +import asyncio +from dataclasses import dataclass +from io import BytesIO + +from pypdf import PdfReader +from pypdf.errors import PdfReadError + +from .contracts import ResolverContentError, TextProjectionContext +from .inspection import ( + ByteContentFacts, + detect_media_type, + format_lexical_facts, + require_bytes, +) +from .label import format_label +from .main import Resolver + + +@dataclass(frozen=True, slots=True) +class PDFSolvedContent(ByteContentFacts): + pdf_version: str | None + page_count: int | None + is_encrypted: bool + title: str | None + author: str | None + + +def _metadata_text(metadata: object, attribute: str) -> str | None: + value = getattr(metadata, attribute, None) + return value if isinstance(value, str) else None + + +def _inspect_pdf(content: bytes) -> PDFSolvedContent: + try: + reader = PdfReader(BytesIO(content), strict=True) + encrypted = reader.is_encrypted + metadata = None if encrypted else reader.metadata + page_count = None if encrypted else len(reader.pages) + except (PdfReadError, OSError, ValueError, TypeError) as error: + raise ResolverContentError("core.pdf.v1", "invalid PDF") from error + + header = reader.pdf_header + version = header.removeprefix("%PDF-") if header.startswith("%PDF-") else None + return PDFSolvedContent( + content=content, + byte_size=len(content), + detected_media_type=detect_media_type(content), + pdf_version=version, + page_count=page_count, + is_encrypted=encrypted, + title=_metadata_text(metadata, "title"), + author=_metadata_text(metadata, "author"), + ) + + +def _extract_pdf_text(content: bytes) -> str | None: + """Extract a bounded text-layer projection without treating metadata as body.""" + try: + reader = PdfReader(BytesIO(content), strict=True) + if reader.is_encrypted: + return None + parts: list[str] = [] + remaining = 1_000_000 + for page in reader.pages[:2_000]: + text = page.extract_text() or "" + if text.strip(): + selected = text[:remaining] + parts.append(selected) + remaining -= len(selected) + if remaining <= 0: + break + except (PdfReadError, OSError, ValueError, TypeError) as error: + raise ResolverContentError("core.pdf.v1", "invalid PDF text layer") from error + joined = "\n\n".join(parts).strip() + return joined or None + + +class PDFResolver( + Resolver[PDFSolvedContent, bytes], + rso_type="core.pdf.v1", +): + async def _get_solved_content( + self, + *, + refresh: bool = False, + materialize_missing: bool = True, + ) -> PDFSolvedContent: + del materialize_missing + content = require_bytes( + await self.get_raw_content(refresh=refresh), + self.__rsotype__, + ) + return await asyncio.to_thread(_inspect_pdf, content) + + async def get_text( + self, + *, + context: TextProjectionContext = "default", + refresh: bool = False, + materialize_missing: bool = True, + ) -> str | None: + del materialize_missing + content = require_bytes( + await self.get_raw_content(refresh=refresh), + self.__rsotype__, + ) + body = await asyncio.to_thread(_extract_pdf_text, content) + if context == "default": + return body + solved = await self.get_solved_content(refresh=refresh, materialize_missing=False) + metadata = format_lexical_facts( + "PDF", + ( + ("media type", solved.detected_media_type), + ("title", solved.title), + ("author", solved.author), + ("pages", solved.page_count), + ("PDF version", solved.pdf_version), + ("encrypted", solved.is_encrypted), + ), + ) + return f"{metadata}\n\n{body}" if body else metadata + + async def get_label(self, *, refresh: bool = False) -> str: + solved = await self.get_solved_content( + refresh=refresh, + materialize_missing=False, + ) + return format_label("PDF", solved.title) diff --git a/inkcre_core/business/info_base/resolver/text.py b/inkcre_core/business/info_base/resolver/text.py new file mode 100644 index 00000000..92feae77 --- /dev/null +++ b/inkcre_core/business/info_base/resolver/text.py @@ -0,0 +1,62 @@ +"""Exact plain-text resolver.""" + +import pydantic + +from .inspection import decode_unicode_bytes +from .label import format_label +from .main import Resolver, TextProjectionContext + + +class TextDraftInput(pydantic.BaseModel): + """Resolver-native input for drafting one plain-text Block.""" + + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + text: str + + +class TextResolver(Resolver[str, str | bytes], rso_type="core.text.v1"): + draft_description = "Create one ordinary plain-text semantic content Block." + draft_input_model = TextDraftInput + + @classmethod + def create_graph(cls, input: str | TextDraftInput): + from inkcre_core.schemas.info_base.block import BlockForm + from inkcre_core.schemas.info_base.main import StarsGraphForm + + text = input.text if isinstance(input, TextDraftInput) else input + return StarsGraphForm(block=BlockForm(resolver=cls.__rsotype__, content=text)) + + async def _get_solved_content( + self, + *, + refresh: bool = False, + materialize_missing: bool = True, + ) -> str: + del materialize_missing + content = await self.get_raw_content(refresh=refresh) + return ( + content + if isinstance(content, str) + else decode_unicode_bytes(content, self.__rsotype__) + ) + + async def get_text( + self, + *, + context: TextProjectionContext = "default", + refresh: bool = False, + materialize_missing: bool = True, + ) -> str: + del context + return await self.get_solved_content( + refresh=refresh, + materialize_missing=materialize_missing, + ) + + async def get_label(self, *, refresh: bool = False) -> str: + return format_label( + "text", + await self.get_text(refresh=refresh, materialize_missing=False), + first_line=True, + ) diff --git a/inkcre_core/business/info_base/resolver/tools.py b/inkcre_core/business/info_base/resolver/tools.py new file mode 100644 index 00000000..84c2fb96 --- /dev/null +++ b/inkcre_core/business/info_base/resolver/tools.py @@ -0,0 +1,277 @@ +"""Agent Tool controller for typed Resolver method discovery and invocation.""" + +import typing + +import pydantic + +from inkcre_core.business.agent import AgentManager +from inkcre_core.business.agent.projection import project_json +from inkcre_core.business.info_base.services import BlockService +from inkcre_core.schemas.ai import JSONValue +from inkcre_core.schemas.info_base.block import BlockID, ResolverType +from inkcre_core.libs.obsrv.telemetry import emit_event + +from .main import ResolverManager + + +RESOLVER_TOOL = "resolver" + + +class ResolverMethodCall(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + block_id: BlockID + method: str + arguments: dict[str, JSONValue] = pydantic.Field(default_factory=dict) + + +class ResolverDescribeInput(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + action: typing.Literal["describe"] + resolver_types: tuple[ResolverType, ...] = () + block_ids: tuple[BlockID, ...] = () + calls: tuple[ResolverMethodCall, ...] = pydantic.Field(default=(), max_length=0) + + +class ResolverInvokeInput(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + action: typing.Literal["invoke"] + resolver_types: tuple[ResolverType, ...] = pydantic.Field(default=(), max_length=0) + block_ids: tuple[BlockID, ...] = pydantic.Field(default=(), max_length=0) + calls: tuple[ResolverMethodCall, ...] = pydantic.Field(min_length=1, max_length=20) + + +class ResolverMetaToolInput( + pydantic.RootModel[ + typing.Annotated[ + ResolverDescribeInput | ResolverInvokeInput, + pydantic.Field(discriminator="action"), + ] + ] +): + pass + + +def _resolver_input_model() -> type[pydantic.BaseModel]: + common = ResolverManager.get_common_method_contracts() + variants: list[type[pydantic.BaseModel]] = [] + contracts = list(common) + common_names = {item.name for item in common} + for resolver_type in ResolverManager.RESOLVER_CLS: + contracts.extend( + contract + for contract in ResolverManager.get_method_contracts(resolver_type) + if contract.name in common_names + ) + seen: set[tuple] = set() + for contract in contracts: + signature = ( + contract.name, + tuple( + (name, repr(field.annotation), repr(field.default), repr(field.metadata)) + for name, field in contract.input_model.model_fields.items() + ), + ) + if signature in seen: + continue + seen.add(signature) + variants.append( + pydantic.create_model( + f"{contract.name}_Call_{len(variants)}", + __config__=pydantic.ConfigDict(extra="forbid"), + block_id=(int, ...), + method=( + typing.cast(typing.Any, typing.Literal)[contract.name], + pydantic.Field(description=contract.description), + ), + arguments=( + contract.input_model, + ... + if any( + field.is_required() for field in contract.input_model.model_fields.values() + ) + else pydantic.Field(default_factory=contract.input_model), + ), + ) + ) + + variants.append( + pydantic.create_model( + "ExtraMethodCall", + __base__=ResolverMethodCall, + method=( + str, + pydantic.Field( + json_schema_extra={"not": {"enum": [contract.name for contract in common]}} + ), + ), + ) + ) + call_type = typing.cast(typing.Any, typing.Union)[tuple(variants)] + invoke = pydantic.create_model( + "BoundResolverInvokeInput", + __base__=ResolverInvokeInput, + calls=(tuple[call_type, ...], pydantic.Field(min_length=1, max_length=20)), + ) + provider_envelope = pydantic.create_model( + "ResolverEnvelope", + __base__=ResolverDescribeInput, + action=(typing.Literal["describe", "invoke"], ...), + calls=(tuple[call_type, ...], pydantic.Field(default=(), max_length=20)), + ) + method_contract = pydantic.RootModel[ + typing.Annotated[ResolverDescribeInput | invoke, pydantic.Field(discriminator="action")] + ] + + # Runtime validates the dispatch envelope, then the selected Resolver validates + # each call's arguments. One invalid method argument must not reject the batch. + # The richer method schema below guides the model without changing that boundary. + class BoundResolverInput(ResolverMetaToolInput): + @classmethod + def model_json_schema(cls, *args, **kwargs) -> dict[str, typing.Any]: + method_schema = method_contract.model_json_schema(*args, **kwargs) + # Some providers infer parameter types only from top-level properties. + # Expose the wider envelope there while retaining the union's oneOf, + # discriminator and $defs so describe/invoke keep their distinct contracts. + provider_schema = provider_envelope.model_json_schema(*args, **kwargs) + method_schema.update(type="object", properties=provider_schema["properties"]) + method_schema.setdefault("$defs", {}).update(provider_schema.get("$defs", {})) + return method_schema + + return BoundResolverInput + + +@AgentManager.tool( + RESOLVER_TOOL, + input_model_factory=_resolver_input_model, + description="Describe or invoke public typed read methods on exact Block Resolvers.", +) +async def resolver(input: ResolverMetaToolInput) -> JSONValue: + request = input.root + if request.action == "describe": + found = await BlockService.get_many(request.block_ids) + resolver_ids = set(request.resolver_types) + resolver_ids.update(block.resolver for block in found) + if not request.block_ids and not request.resolver_types: + resolver_ids.update(ResolverManager.RESOLVER_CLS) + return typing.cast( + JSONValue, + { + "results": [ + { + "resolver": resolver_id, + "methods": [ + { + "name": contract.name, + "description": contract.description, + "input_schema": contract.input_schema, + } + for contract in ResolverManager.get_method_contracts(resolver_id) + ], + } + for resolver_id in sorted(resolver_ids) + if resolver_id in ResolverManager.RESOLVER_CLS + ], + "missing_blocks": sorted(set(request.block_ids) - {block.id for block in found}), + "missing_resolvers": sorted( + resolver_id + for resolver_id in resolver_ids + if resolver_id not in ResolverManager.RESOLVER_CLS + ), + }, + ) + + results: list[JSONValue] = [] + for index, call in enumerate(request.calls): + block = await BlockService.get(call.block_id) + if block is None: + results.append( + { + "index": index, + "block_id": call.block_id, + "method": call.method, + "error": "not_found", + } + ) + continue + contract = ResolverManager.get_method_contract(block.resolver, call.method) + if block.resolver not in ResolverManager.RESOLVER_CLS: + results.append( + { + "index": index, + "block_id": call.block_id, + "method": call.method, + "error": "resolver_unavailable", + "message": f"Resolver {block.resolver!r} is not registered.", + } + ) + continue + if contract is None: + results.append( + { + "index": index, + "block_id": call.block_id, + "method": call.method, + "error": "method_unavailable", + "message": "Method does not exist; use describe for available method contracts.", + "available_methods": [ + item.name for item in ResolverManager.get_method_contracts(block.resolver) + ], + } + ) + continue + try: + value = await ResolverManager.invoke_method( + block, + call.method, + call.arguments.model_dump() + if isinstance(call.arguments, pydantic.BaseModel) + else typing.cast(dict[str, typing.Any], call.arguments), + ) + projected = project_json(value) + except pydantic.ValidationError as error: + results.append( + typing.cast( + JSONValue, + { + "index": index, + "block_id": call.block_id, + "method": call.method, + "error": "invalid_arguments", + "fields": error.errors( + include_url=False, include_context=False, include_input=False + ), + "input_schema": contract.input_schema, + }, + ) + ) + except Exception as error: + results.append( + { + "index": index, + "block_id": call.block_id, + "method": call.method, + "error": type(error).__name__, + "message": str(error), + } + ) + else: + emit_event( + "inkcre.entity.read", + { + "inkcre.entity.read.kind": "resolved", + "inkcre.entity.block_ids": (call.block_id,), + "inkcre.entity.read.count": 1, + }, + ) + results.append( + { + "index": index, + "block_id": call.block_id, + "method": call.method, + "result": projected, + } + ) + return typing.cast(JSONValue, {"results": results}) diff --git a/inkcre_core/business/info_base/resolver/video.py b/inkcre_core/business/info_base/resolver/video.py new file mode 100644 index 00000000..5a684c37 --- /dev/null +++ b/inkcre_core/business/info_base/resolver/video.py @@ -0,0 +1,204 @@ +"""Exact video resolver using metadata-only PyAV inspection.""" + +import asyncio +from dataclasses import dataclass +from fractions import Fraction +from io import BytesIO +import typing + +import av +import pydantic + +from inkcre_core.business.deployment_config import ( + DeploymentConfigManager, + DeploymentConfigService, +) +from inkcre_core.schemas.ai import VideoContentPart +from inkcre_core.schemas.info_base.block import BlockForm +from inkcre_core.schemas.info_base.main import StarsGraphForm + +from .audio import _container_name, _duration_ms, _primary_stream +from .contracts import ( + ResolverContentError, + TextProjectionContext, + UnsupportedResolverCapability, +) +from .inspection import ( + ByteContentFacts, + detect_media_type, + format_lexical_facts, + require_bytes, +) +from .main import Resolver +from .materialization import materialize_text_child, try_materialize_model_text + + +VIDEO_RESOLVER_CONFIG_KEY = "core.resolver.video" +VIDEO_RESOLVER_CONFIG_SCHEMA = "core.resolver.video.config.v1" + + +class VideoResolverConfig(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + text_model: int + transcript_model: int + + +DeploymentConfigManager.register_schema( + VIDEO_RESOLVER_CONFIG_SCHEMA, VideoResolverConfig, keys=(VIDEO_RESOLVER_CONFIG_KEY,) +) + + +@dataclass(frozen=True, slots=True) +class VideoSolvedContent(ByteContentFacts): + container: str | None + video_codec: str | None + duration_ms: int | None + width: int | None + height: int | None + frame_rate: float | None + subtitles: tuple[str, ...] + + +def _frame_rate(rate: Fraction | None) -> float | None: + return float(rate) if rate is not None else None + + +def _subtitles(container: typing.Any) -> tuple[str, ...]: + values: list[str] = [] + streams = tuple(container.streams.subtitles) + if not streams: + return () + for packet in container.demux(*streams): + try: + decoded = packet.decode() + except (av.FFmpegError, OSError, ValueError): + continue + for subtitle in decoded: + raw = getattr(subtitle, "dialogue", None) or getattr(subtitle, "text", None) + if isinstance(raw, bytes): + text = raw.decode("utf-8", errors="replace").strip() + elif isinstance(raw, str): + text = raw.strip() + else: + text = "" + if text and (not values or values[-1] != text): + values.append(text) + return tuple(values) + + +def _inspect_video(content: bytes) -> VideoSolvedContent: + try: + with av.open(BytesIO(content), mode="r") as container: + stream = _primary_stream(container.streams.video) + if stream is None: + raise ResolverContentError("core.video.v1", "no video stream") + codec = stream.codec_context + facts = VideoSolvedContent( + content=content, + byte_size=len(content), + detected_media_type=detect_media_type(content), + container=_container_name(container.format.name), + video_codec=codec.name, + duration_ms=_duration_ms(container, stream), + width=codec.width, + height=codec.height, + frame_rate=_frame_rate(stream.average_rate), + subtitles=_subtitles(container), + ) + except ResolverContentError: + raise + except (av.FFmpegError, OSError, ValueError) as error: + raise ResolverContentError("core.video.v1", "invalid video") from error + return facts + + +class VideoResolver( + Resolver[VideoSolvedContent, bytes], + rso_type="core.video.v1", +): + @classmethod + def create_graph(cls, url: str) -> StarsGraphForm: + return StarsGraphForm( + block=BlockForm(resolver=cls.__rsotype__, content=url, storage=-1) + ) + + async def _get_solved_content( + self, + *, + refresh: bool = False, + materialize_missing: bool = True, + ) -> VideoSolvedContent: + del materialize_missing + content = require_bytes( + await self.get_raw_content(refresh=refresh), + self.__rsotype__, + ) + return await asyncio.to_thread(_inspect_video, content) + + async def get_text( + self, + *, + context: TextProjectionContext = "default", + refresh: bool = False, + materialize_missing: bool = True, + ) -> str: + if context == "default": + raise UnsupportedResolverCapability(self.__rsotype__, "text") + solved = await self.get_solved_content(refresh=refresh, materialize_missing=False) + if materialize_missing: + if solved.subtitles: + await materialize_text_child( + self.block_id, "subtitle", "\n\n".join(solved.subtitles) + ) + config = typing.cast( + VideoResolverConfig | None, + await DeploymentConfigService.get(VIDEO_RESOLVER_CONFIG_KEY), + ) + if config is not None: + media_type = solved.detected_media_type or ( + f"video/{solved.container}" if solved.container is not None else None + ) + if media_type is not None: + media = VideoContentPart( + data=solved.content, + mime_type=media_type, + transfer_url=await self.get_transfer_url(), + ) + await try_materialize_model_text( + block_id=self.block_id, + role="transcript", + model=config.transcript_model, + instruction=( + "Transcribe only speech and authored spoken language in the video. " + "Return plain text with no summary or visual description." + ), + media=media, + ) + await try_materialize_model_text( + block_id=self.block_id, + role="text", + model=config.text_model, + instruction=( + "Transcribe only written text visibly present in the video. " + "Return plain text with no summary or scene description." + ), + media=media, + ) + return format_lexical_facts( + "video", + ( + ("media type", solved.detected_media_type), + ("container", solved.container), + ("codec", solved.video_codec), + ("duration ms", solved.duration_ms), + ("width", solved.width), + ("height", solved.height), + ("frame rate", solved.frame_rate), + ("subtitle cues", len(solved.subtitles) or None), + ), + ) + + async def get_label(self, *, refresh: bool = False) -> str: + del refresh + return "video" diff --git a/inkcre_core/business/info_base/resolver/zip.py b/inkcre_core/business/info_base/resolver/zip.py new file mode 100644 index 00000000..0a0a0f35 --- /dev/null +++ b/inkcre_core/business/info_base/resolver/zip.py @@ -0,0 +1,86 @@ +"""Exact ZIP resolver using central-directory metadata only.""" + +import asyncio +from dataclasses import dataclass +from io import BytesIO +import zipfile + +from .contracts import ( + ResolverContentError, + TextProjectionContext, + UnsupportedResolverCapability, +) +from .inspection import ByteContentFacts, format_lexical_facts, require_bytes +from .main import Resolver + + +@dataclass(frozen=True, slots=True) +class ZIPSolvedContent(ByteContentFacts): + member_count: int + total_compressed_bytes: int + total_uncompressed_bytes: int + compression_methods: tuple[int, ...] + encrypted_member_count: int + + +def _inspect_zip(content: bytes) -> ZIPSolvedContent: + try: + with zipfile.ZipFile(BytesIO(content)) as archive: + members = archive.infolist() + except (zipfile.BadZipFile, OSError, ValueError) as error: + raise ResolverContentError("core.zip.v1", "invalid ZIP archive") from error + + return ZIPSolvedContent( + content=content, + byte_size=len(content), + detected_media_type="application/zip", + member_count=len(members), + total_compressed_bytes=sum(member.compress_size for member in members), + total_uncompressed_bytes=sum(member.file_size for member in members), + compression_methods=tuple(sorted({member.compress_type for member in members})), + encrypted_member_count=sum(bool(member.flag_bits & 0x1) for member in members), + ) + + +class ZIPResolver( + Resolver[ZIPSolvedContent, bytes], + rso_type="core.zip.v1", +): + async def _get_solved_content( + self, + *, + refresh: bool = False, + materialize_missing: bool = True, + ) -> ZIPSolvedContent: + del materialize_missing + content = require_bytes( + await self.get_raw_content(refresh=refresh), + self.__rsotype__, + ) + return await asyncio.to_thread(_inspect_zip, content) + + async def get_text( + self, + *, + context: TextProjectionContext = "default", + refresh: bool = False, + materialize_missing: bool = True, + ) -> str: + if context == "default": + raise UnsupportedResolverCapability(self.__rsotype__, "text") + del materialize_missing + solved = await self.get_solved_content(refresh=refresh, materialize_missing=False) + return format_lexical_facts( + "ZIP", + ( + ("media type", solved.detected_media_type), + ("members", solved.member_count), + ("compressed bytes", solved.total_compressed_bytes), + ("uncompressed bytes", solved.total_uncompressed_bytes), + ("encrypted members", solved.encrypted_member_count), + ), + ) + + async def get_label(self, *, refresh: bool = False) -> str: + del refresh + return "ZIP" diff --git a/inkcre_core/business/info_base/services.py b/inkcre_core/business/info_base/services.py new file mode 100644 index 00000000..5facb4bc --- /dev/null +++ b/inkcre_core/business/info_base/services.py @@ -0,0 +1,183 @@ +"""Independent info-base use cases; composed writes use GraphUnitOfWork directly.""" + +import typing +from typing import Optional as Opt +from inkcre_core.schemas.info_base.block import BlockForm, BlockModel, BlockID, ResolverType +from inkcre_core.schemas.info_base.relation import RelationModel, RelationID +from inkcre_core.schemas.info_base.storage import StorageID +from inkcre_core.utils.types_ import Undefined, _undefined +from inkcre_core.persistence.info_base.uow import graph_uow + + +class BlockService: + @staticmethod + async def get_many(block_ids: typing.Collection[BlockID]) -> tuple[BlockModel, ...]: + async with graph_uow() as uow: + result = await uow.blocks.get_many(block_ids) + return result + + @staticmethod + async def get_random_many(count: int) -> tuple[BlockModel, ...]: + async with graph_uow() as uow: + result = await uow.blocks.get_random_many(count) + return result + + @staticmethod + async def get_random() -> Opt[BlockModel]: + async with graph_uow() as uow: + result = await uow.blocks.get_random() + return result + + @staticmethod + async def get(block_id: BlockID) -> Opt[BlockModel]: + async with graph_uow() as uow: + result = await uow.blocks.get(block_id) + return result + + @staticmethod + async def edit_block( + block_id: BlockID, + content: Opt[str] = None, + resolver: Opt[ResolverType] = None, + storage: Opt[StorageID] | Undefined = _undefined, + ) -> BlockModel: + async with graph_uow() as uow: + result = await uow.blocks.edit_block(block_id, content, resolver, storage) + return result + + @staticmethod + async def delete(block_id: BlockID) -> bool: + async with graph_uow() as uow: + result = await uow.blocks.delete(block_id) + return result + + @staticmethod + async def get_recent( + num: int = 10, resolver: Opt[ResolverType] = None + ) -> tuple[BlockModel, ...]: + """获取最新的块 + + 按创建时间倒序排序。 + + :param num: 获取的块数量 + :param resolver: 限定解析器类型, None则不限定 + """ + async with graph_uow() as uow: + result = await uow.blocks.get_recent(num, resolver) + return result + + @staticmethod + async def create(form: BlockForm) -> BlockModel: + async with graph_uow() as uow: + result = await uow.blocks.create(form) + return result + + +class RelationService: + @staticmethod + async def get_text( + relation: RelationModel, + *, + refresh: bool = False, + ) -> str | None: + """Project one directed dynamic property through Block-local endpoint labels.""" + if not relation.content.strip(): + return None + async with graph_uow() as uow: + endpoints = { + block.id: block + for block in await uow.blocks.get_many((relation.from_, relation.to_)) + } + from_block = endpoints.get(relation.from_) + to_block = endpoints.get(relation.to_) + if from_block is None or to_block is None: + return None + + # Local imports avoid reversing Resolver -> RelationService ownership. + from inkcre_core.business.info_base.resolver import ResolverManager + + subject = await ResolverManager.get(from_block).get_label(refresh=refresh) + value = await ResolverManager.get(to_block).get_label(refresh=refresh) + if not subject.strip() or not value.strip(): + return None + return f"subject:\n{subject}\nproperty:\n{relation.content}\nvalue:\n{value}" + + @staticmethod + async def get_many( + relation_ids: typing.Collection[RelationID], + ) -> tuple[RelationModel, ...]: + async with graph_uow() as uow: + result = await uow.relations.get_many(relation_ids) + return result + + @staticmethod + async def get_by_id(relation_id: RelationID) -> RelationModel | None: + async with graph_uow() as uow: + result = await uow.relations.get_by_id(relation_id) + return result + + @staticmethod + async def get_endpoint_page( + block_ids: typing.Collection[BlockID], + *, + endpoint: typing.Literal["from", "to"], + contents: typing.Collection[str] = (), + cursor: RelationID | None = None, + limit: int = 21, + ) -> tuple[RelationModel, ...]: + async with graph_uow() as uow: + result = await uow.relations.get_endpoint_page( + block_ids, endpoint=endpoint, contents=contents, cursor=cursor, limit=limit + ) + return result + + @staticmethod + async def get( + block_id: BlockID, + include_in: bool = True, + include_out: bool = True, + content: Opt[str] = None, + ) -> tuple[RelationModel, ...]: + async with graph_uow() as uow: + result = await uow.relations.get(block_id, include_in, include_out, content) + return result + + @staticmethod + async def update( + relation_id: RelationID, + *, + from_: BlockID | Undefined = _undefined, + to_: BlockID | Undefined = _undefined, + content: str | Undefined = _undefined, + ) -> RelationModel: + async with graph_uow() as uow: + result = await uow.relations.update( + relation_id, from_=from_, to_=to_, content=content + ) + return result + + @staticmethod + async def delete(relation_id: RelationID) -> bool: + async with graph_uow() as uow: + result = await uow.relations.delete(relation_id) + return result + + @staticmethod + async def create(from_: BlockID, to_: BlockID, content: str) -> RelationModel: + async with graph_uow() as uow: + result = await uow.relations.create(from_, to_, content) + return result + + +async def get_entity_records( + block_ids: typing.Collection[BlockID], relation_ids: typing.Collection[RelationID] +) -> tuple[dict[BlockID, BlockModel], dict[RelationID, RelationModel]]: + """Read both entity kinds in one short scope before any content hydration.""" + async with graph_uow() as uow: + blocks = await uow.blocks.get_many(block_ids) + relations = await uow.relations.get_many(relation_ids) + # Persisted rows always have primary keys; the shared model also permits new rows. + return ( + {typing.cast(BlockID, block.id): block for block in blocks}, + {typing.cast(RelationID, row.id): row for row in relations}, + ) diff --git a/app/business/info_base/storage/AGENTS.md b/inkcre_core/business/info_base/storage/AGENTS.md similarity index 97% rename from app/business/info_base/storage/AGENTS.md rename to inkcre_core/business/info_base/storage/AGENTS.md index 8e26a194..41edc0ce 100644 --- a/app/business/info_base/storage/AGENTS.md +++ b/inkcre_core/business/info_base/storage/AGENTS.md @@ -1,6 +1,6 @@ # storage/ Local Guide -本文件只描述 `app/business/info_base/storage/` 的局部事实与风险边界。上层 contract 看 +本文件只描述 `inkcre_core/business/info_base/storage/` 的局部事实与风险边界。上层 contract 看 [info_base guide](../AGENTS.md)。 ## 何时阅读 diff --git a/inkcre_core/business/info_base/storage/__init__.py b/inkcre_core/business/info_base/storage/__init__.py new file mode 100644 index 00000000..2f318982 --- /dev/null +++ b/inkcre_core/business/info_base/storage/__init__.py @@ -0,0 +1,15 @@ +from .main import Storage, StorageManager, WritableStorage + +__all__ = [ + "Storage", + "StorageManager", + "WritableStorage", + "HTTPStorage", + "HTTPStorageConfig", + "StorageContentTooLargeError", + "PostgreSQLBinaryStorage", +] + +# Import storage implementations to ensure they're registered +from .http import HTTPStorage, HTTPStorageConfig, StorageContentTooLargeError +from .postgresql import PostgreSQLBinaryStorage diff --git a/inkcre_core/business/info_base/storage/http.py b/inkcre_core/business/info_base/storage/http.py new file mode 100644 index 00000000..61b35592 --- /dev/null +++ b/inkcre_core/business/info_base/storage/http.py @@ -0,0 +1,86 @@ +"""Mechanics-only HTTP storage returning bounded response bytes.""" + +from __future__ import annotations + +import aiohttp +import pydantic +import sqlmodel + +from inkcre_core.utils.base import AIOHTTP_CONNECTOR_GETTER + +from .main import Storage + + +__all__ = [ + "HTTPStorageConfig", + "HTTPStorage", + "StorageContentTooLargeError", +] + + +DEFAULT_MAX_RESPONSE_BYTES = 64 * 1024 * 1024 +READ_CHUNK_BYTES = 64 * 1024 + + +class HTTPStorageConfig(sqlmodel.SQLModel): + """Configuration for bounded HTTP byte retrieval.""" + + timeout: int = pydantic.Field(default=30, gt=0) + follow_redirects: bool = True + max_response_bytes: int = pydantic.Field( + default=DEFAULT_MAX_RESPONSE_BYTES, + gt=0, + ) + + +class StorageContentTooLargeError(ValueError): + """The configured byte boundary was exceeded while hydrating content.""" + + +class HTTPStorage( + Storage[HTTPStorageConfig, bytes], + stg_type="http", + config_cls=HTTPStorageConfig, +): + """Fetch opaque bytes over HTTP(S) without assigning content semantics.""" + + @staticmethod + def _url(block_content: str) -> str: + url = block_content.strip() + if not url.lower().startswith(("http://", "https://")): + raise ValueError("HTTP storage pointer must use http:// or https://") + return url + + def get_transfer_url(self, block_content: str) -> str: + return self._url(block_content) + + async def get_raw_content(self, block_content: str) -> bytes: + url = self._url(block_content) + + timeout = aiohttp.ClientTimeout(total=self._config.timeout) + async with aiohttp.ClientSession( + connector=AIOHTTP_CONNECTOR_GETTER(), + timeout=timeout, + ) as session: + async with session.get( + url, + allow_redirects=self._config.follow_redirects, + ) as response: + response.raise_for_status() + content_length = response.content_length + if content_length is not None and content_length > self._config.max_response_bytes: + raise StorageContentTooLargeError( + f"HTTP response declares {content_length} bytes; " + f"limit is {self._config.max_response_bytes}" + ) + + chunks: list[bytes] = [] + received = 0 + async for chunk in response.content.iter_chunked(READ_CHUNK_BYTES): + received += len(chunk) + if received > self._config.max_response_bytes: + raise StorageContentTooLargeError( + f"HTTP response exceeded {self._config.max_response_bytes} bytes" + ) + chunks.append(chunk) + return b"".join(chunks) diff --git a/inkcre_core/business/info_base/storage/main.py b/inkcre_core/business/info_base/storage/main.py new file mode 100644 index 00000000..95f84227 --- /dev/null +++ b/inkcre_core/business/info_base/storage/main.py @@ -0,0 +1,186 @@ +__all__ = [ + "StorageManager", + "Storage", +] + +import abc +import importlib +import sqlmodel +import typing +from inkcre_core.persistence.info_base.storage import StorageRepository +from inkcre_core.database_contract.profile import ( + BUILTIN_STORAGES, + BUILTIN_STORAGE_TYPES_BY_ID, +) +from inkcre_core.schemas.info_base.storage import ( + StorageID, + StorageTypeID, + StorageModel, +) + + +ConfigTV = typing.TypeVar("ConfigTV", bound=sqlmodel.SQLModel) +ContentTV = typing.TypeVar("ContentTV") + + +class _EmptyConfig(sqlmodel.SQLModel): + """Default empty config class for storages without configuration.""" + + +class StorageManager: + _STORAGE_CLASSES: dict[StorageTypeID, type["Storage"]] = {} + """Storage type registry: type ID -> Storage class""" + + @classmethod + def register_storage(cls, storage_cls: type["Storage"]): + """Register a storage class in memory without external side effects.""" + cls._STORAGE_CLASSES[storage_cls.__stgtype__] = storage_cls + + @classmethod + async def get_storage_async(cls, storage_id: StorageID) -> "Storage": + from inkcre_core.persistence.info_base.uow import graph_uow + + async with graph_uow() as uow: + record = await uow.storage.get(storage_id) + return cls.from_record(record) + + @classmethod + def from_record(cls, record: StorageModel) -> "Storage": + storage_class = cls._STORAGE_CLASSES.get(record.type) + if storage_class is None: + module_path, class_name = record.type.rsplit(".", 1) + storage_class = getattr(importlib.import_module(module_path), class_name) + return storage_class(record) + + @classmethod + async def setup_builtin_storages_async(cls) -> None: + from inkcre_core.persistence.info_base.uow import graph_uow + + records = [] + for storage_cls in cls._STORAGE_CLASSES.values(): + builtin = BUILTIN_STORAGE_TYPES_BY_ID.get(storage_cls.__stgtype__) + records.append( + { + "id": storage_cls.__stgtype__, + "description": builtin.description + if builtin + else storage_cls.__doc__ or "No description.", + "config_schema": builtin.config_schema + if builtin + else storage_cls.__configschema__, + "writable": issubclass(storage_cls, WritableStorage), + } + ) + async with graph_uow() as uow: + await uow.storage.sync_types(records) + await uow.storage.sync_builtins( + [ + {"id": row.id, "type": row.type, "nickname": row.nickname, "config": row.config} + for row in BUILTIN_STORAGES + ] + ) + + +class Storage(abc.ABC, typing.Generic[ConfigTV, ContentTV]): + """Storage base. + Storage retrieves the raw ("real") content from block record. + + Generic parameters: + - ConfigTV: Configuration type variable + - ContentTV: Content type variable + """ + + __configschema__: dict + """Storage configuration JSON schema""" + __configcls__: type[ConfigTV] + __stgtype__: StorageTypeID + """Storage type identifier""" + + def __init_subclass__( + cls, + stg_type: StorageTypeID | None = None, + config_cls: type[ConfigTV] = _EmptyConfig, + **kwargs, + ) -> None: + """ + :param stg_type: Unique storage type string + :param config_cls: Configuration class for the storage + """ + # ConfigTV is bound by the concrete storage subclass. + if stg_type is not None: + cls.__configcls__ = config_cls # pyrefly: ignore[no-access] + cls.__configschema__ = config_cls.model_json_schema() + cls.__stgtype__ = stg_type + StorageManager.register_storage(cls) + return super().__init_subclass__(**kwargs) + + def __init__(self, storage_record: StorageModel): + if storage_record.id is None: + raise ValueError("Storage must be persisted before use") + self._id = storage_record.id + self._config = self.__configcls__.model_validate(storage_record.config) + + self.__post_init__() + + @property + def storage_id(self) -> StorageID: + """Return the persisted Storage reference represented by this instance.""" + return self._id + + def __post_init__(self): + """Post-initialization hook for subclasses.""" + pass + + async def get_raw_content(self, block_content: str) -> ContentTV: + """Get the raw content of the block.""" + raise NotImplementedError( + f"{self.__class__.__name__}.get_raw_content() must be implemented by subclasses." + ) + + def get_transfer_url(self, block_content: str) -> str | None: + """Return an optional external byte-transfer hint for AI/provider use. + + The URL is neither content authority nor a promise that every external + provider can fetch it. Callers must retain hydrated content as the primary + value and treat this result as an optimization hint. + """ + del block_content + return None + + +class WritableStorage(Storage[ConfigTV, ContentTV], abc.ABC): + """Storage capability for raw content owned by the current deployment.""" + + async def get_raw_content(self, block_content: str) -> ContentTV: + from inkcre_core.persistence.info_base.uow import graph_uow + + async with graph_uow() as uow: + return await self.read_content(block_content, uow.storage) + + async def create_content(self, content: ContentTV, storage: StorageRepository) -> str: + return self.serialize_pointer(await self.write_content(content, storage)) + + @abc.abstractmethod + async def read_content( + self, block_content: str, storage: StorageRepository + ) -> ContentTV: ... + + @abc.abstractmethod + async def write_content( + self, content: ContentTV, storage: StorageRepository + ) -> typing.Any: ... + + @abc.abstractmethod + async def update_content( + self, block_content: str, content: ContentTV, storage: StorageRepository + ) -> bool: ... + + @abc.abstractmethod + async def delete_content( + self, block_content: str, storage: StorageRepository + ) -> bool: ... + + @abc.abstractmethod + def serialize_pointer(self, pointer: typing.Any) -> str: + """Encode a storage-owned key as one opaque block pointer string.""" + ... diff --git a/inkcre_core/business/info_base/storage/postgresql.py b/inkcre_core/business/info_base/storage/postgresql.py new file mode 100644 index 00000000..0c11bb00 --- /dev/null +++ b/inkcre_core/business/info_base/storage/postgresql.py @@ -0,0 +1,59 @@ +"""Writable PostgreSQL binary storage.""" + +import uuid + +import pydantic +import sqlmodel + +from .main import WritableStorage +from inkcre_core.persistence.info_base.storage import StorageRepository + + +class PostgreSQLBinaryStorageConfig(sqlmodel.SQLModel): + pass + + +class PostgreSQLBlobPointer(pydantic.BaseModel): + """Minimum pointer fields accepted from storage-backed block content.""" + + model_config = pydantic.ConfigDict(extra="ignore") + + blob_id: uuid.UUID + + +class StorageBlobNotFoundError(LookupError): + pass + + +class PostgreSQLBinaryStorage( + WritableStorage[PostgreSQLBinaryStorageConfig, bytes], + stg_type="postgresql_binary", + config_cls=PostgreSQLBinaryStorageConfig, +): + """Store raw bytes in the protocol database and return an opaque UUID pointer.""" + + def serialize_pointer(self, pointer: object) -> str: + blob_id = pydantic.TypeAdapter(uuid.UUID).validate_python(pointer) + return PostgreSQLBlobPointer(blob_id=blob_id).model_dump_json() + + async def read_content( + self, block_content: str, storage: StorageRepository, *, lock: bool = False + ) -> bytes: + pointer = PostgreSQLBlobPointer.model_validate_json(block_content) + content = await storage.read_blob(pointer.blob_id, lock=lock) + if content is None: + raise StorageBlobNotFoundError(f"Storage blob {pointer.blob_id} not found") + return content + + async def write_content(self, content: bytes, storage: StorageRepository) -> uuid.UUID: + return await storage.create_blob(content) + + async def update_content( + self, block_content: str, content: bytes, storage: StorageRepository + ) -> bool: + pointer = PostgreSQLBlobPointer.model_validate_json(block_content) + return await storage.update_blob(pointer.blob_id, content) + + async def delete_content(self, block_content: str, storage: StorageRepository) -> bool: + pointer = PostgreSQLBlobPointer.model_validate_json(block_content) + return await storage.delete_blob(pointer.blob_id) diff --git a/inkcre_core/business/info_base/tools.py b/inkcre_core/business/info_base/tools.py new file mode 100644 index 00000000..08fe5d14 --- /dev/null +++ b/inkcre_core/business/info_base/tools.py @@ -0,0 +1,152 @@ +"""Agent Tool controllers for info-base retrieval and entity reads.""" + +import typing + +import pydantic + +from inkcre_core.business.agent import AgentManager +from inkcre_core.business.agent.projection import project_json +from inkcre_core.schemas.ai import JSONValue +from inkcre_core.schemas.info_base.block import BlockModel +from inkcre_core.schemas.info_base.relation import RelationModel +from inkcre_core.schemas.lexical_retrieval import LexicalRetrievalResult +from inkcre_core.libs.obsrv.telemetry import emit_event + +from .main import InfoBaseManager + + +RETRIEVE_TOOL = "retrieve" +GET_ENTITIES_TOOL = "get_entities" +RetrievalMode: typing.TypeAlias = typing.Literal["lexical", "semantic", "hybrid"] + + +class RetrieveInput(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + query: str + mode: RetrievalMode = "hybrid" + limit: int = pydantic.Field(default=20, ge=1, le=20) + + @pydantic.field_validator("query") + @classmethod + def non_empty_query(cls, value: str) -> str: + if not value.strip(): + raise ValueError("query must not be empty") + return value + + +class EntityReference(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + type: typing.Literal["block", "relation"] + id: int + + +class GetEntitiesInput(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + entities: tuple[EntityReference, ...] = pydantic.Field(default=(), max_length=20) + random_count: int = pydantic.Field(default=1, ge=1, le=20) + + @pydantic.model_validator(mode="after") + def validate_selection(self) -> typing.Self: + if self.entities and self.random_count != 1: + raise ValueError("random_count only applies when entities is empty") + return self + + +@AgentManager.tool( + RETRIEVE_TOOL, + description="Retrieve lexical, semantic, or separate hybrid results for one query.", +) +async def retrieve(input: RetrieveInput) -> JSONValue: + branches = await InfoBaseManager.retrieve(input.query, input.mode, input.limit) + payload: dict[str, JSONValue] = {} + for name, branch in branches.items(): + if branch.error is not None: + payload[name] = { + "error": type(branch.error).__name__, + "message": str(branch.error), + } + continue + result = branch.result + assert result is not None + if isinstance(result, LexicalRetrievalResult): + block_ids = tuple( + match.block.id for match in result.matches if match.block.id is not None + ) + relation_ids: tuple[int, ...] = () + else: + block_ids = tuple( + match.entity.id + for match in result.matches + if match.type == "block" and match.entity.id is not None + ) + relation_ids = tuple( + match.entity.id + for match in result.matches + if match.type == "relation" and match.entity.id is not None + ) + emit_event( + "inkcre.retrieval.candidates", + { + "inkcre.retrieval.method": "lexical" if name == "lexical" else "semantic", + "inkcre.entity.block_ids": block_ids, + "inkcre.entity.relation_ids": relation_ids, + "inkcre.retrieval.candidate_count": len(result.matches), + }, + ) + if name == "lexical": + payload[name] = { + "matches": [ + { + "entity": {"entity_type": "block", "entity_id": match.block.id}, + **match.model_dump(mode="json", exclude={"block"}), + } + for match in typing.cast(typing.Any, result).matches + ] + } + else: + payload[name] = { + **result.model_dump(mode="json", exclude={"matches"}), + "matches": [ + { + "entity": {"entity_type": match.type, "entity_id": match.entity.id}, + "score": match.score, + } + for match in typing.cast(typing.Any, result).matches + ], + } + return payload + + +@AgentManager.tool( + GET_ENTITIES_TOOL, + description=( + "Read persisted Blocks or Relations without resolving content. " + "Null may indicate an incorrect entity type." + ), +) +async def get_entities(input: GetEntitiesInput) -> JSONValue: + result = await InfoBaseManager.get_entities( + tuple((reference.type, reference.id) for reference in input.entities), + random_count=input.random_count, + ) + emit_event( + "inkcre.entity.read", + { + "inkcre.entity.read.kind": "persisted", + "inkcre.entity.block_ids": tuple( + entity.id + for entity in result + if isinstance(entity, BlockModel) and entity.id is not None + ), + "inkcre.entity.relation_ids": tuple( + entity.id + for entity in result + if isinstance(entity, RelationModel) and entity.id is not None + ), + "inkcre.entity.read.count": sum(entity is not None for entity in result), + }, + ) + return project_json(result) diff --git a/inkcre_core/business/job.py b/inkcre_core/business/job.py new file mode 100644 index 00000000..2459a0ce --- /dev/null +++ b/inkcre_core/business/job.py @@ -0,0 +1,422 @@ +"""Deep runtime for typed durable background Jobs.""" + +import abc +import asyncio +import dataclasses +import typing + +import jsonschema # pyrefly: ignore[untyped-import] +import pydantic + +from inkcre_core.database_contract.profile import BUILTIN_JOB_TYPES_BY_ID +from inkcre_core.persistence.job.uow import JobUnitOfWork, job_uow +from inkcre_core.context import current_core +from inkcre_core.libs.obsrv.log_record import ENABLE_LOG_BACKEND, TRACE_ID +from inkcre_core.schemas.job import JobID, JobModel, JobStatus, JobTypeID, JobTypeModel +from inkcre_core.libs.obsrv.main import get_logger +from inkcre_core.libs.obsrv.telemetry import emit_event, operation +from inkcre_core.libs.obsrv.propagation import submission_context, submission_links +from opentelemetry.context import Context + + +LOGGER = get_logger().getChild(__name__) +ParametersTV = typing.TypeVar("ParametersTV", bound=pydantic.BaseModel) + + +class UnknownJobTypeError(ValueError): + """A submitted exact Job type is absent from the deployment catalog.""" + + +@dataclasses.dataclass +class _Execution: + task: asyncio.Task + cancellation_requested: bool = False + closing: bool = False + + def cancel(self) -> None: + # Do not interrupt cleanup already started by timeout or an earlier abort. + if not self.cancellation_requested and not self.closing: + self.cancellation_requested = True + if not self.task.cancelling(): + self.task.cancel() + + +class JobHandler(abc.ABC, typing.Generic[ParametersTV]): + """One exact executable Job contract registered by its owning module.""" + + type: typing.ClassVar[JobTypeID] + description: typing.ClassVar[str] + parameters_model: typing.ClassVar[type[pydantic.BaseModel]] + default_timeout_seconds: typing.ClassVar[int] + + def __init_subclass__( + cls, + *, + job_type: JobTypeID, + description: str, + parameters_model: type[ParametersTV], + default_timeout_seconds: int, + **kwargs, + ) -> None: + if default_timeout_seconds <= 0: + raise ValueError("default_timeout_seconds must be positive") + cls.type = job_type + cls.description = description + cls.parameters_model = parameters_model + cls.default_timeout_seconds = default_timeout_seconds + JobManager.register_handler(cls) + super().__init_subclass__(**kwargs) + + @classmethod + def validate_parameters(cls, parameters: dict[str, typing.Any]) -> ParametersTV: + return typing.cast(ParametersTV, cls.parameters_model.model_validate(parameters)) + + @classmethod + async def normalize_parameters( + cls, parameters: dict[str, typing.Any], uow: JobUnitOfWork + ) -> dict[str, typing.Any]: + """Validate a new Job/Cron input; owners may resolve nested catalog contracts. + + Execution restores the persisted model through validate_parameters instead; + it does not repeat submission-only normalization. + """ + del uow + return cls.validate_parameters(parameters).model_dump(mode="json") + + @classmethod + @abc.abstractmethod + async def can_handle(cls, parameters: ParametersTV) -> bool: + """Return whether this runtime can execute these parameters now.""" + ... + + @classmethod + @abc.abstractmethod + async def handle(cls, job: JobModel, parameters: ParametersTV) -> None: + """Execute one already claimed Job.""" + ... + + +class JobManager: + """Own Job Handler registration, typed creation, claim and terminal closure.""" + + _handlers: dict[JobTypeID, type[JobHandler]] = {} + + @classmethod + def register_handler(cls, handler: type[JobHandler]) -> None: + existing = cls._handlers.get(handler.type) + if existing is handler: + return + if existing is not None: + raise ValueError( + f"Job type {handler.type!r} is already handled by " + f"{existing.__module__}.{existing.__qualname__}" + ) + cls._handlers[handler.type] = handler + + @classmethod + async def sync_job_types(cls) -> None: + rows = [] + for handler in cls._handlers.values(): + builtin = BUILTIN_JOB_TYPES_BY_ID.get(handler.type) + rows.append( + dict( + id=handler.type, + description=builtin.description if builtin is not None else handler.description, + parameters_schema=( + builtin.parameters_schema + if builtin is not None + else handler.parameters_model.model_json_schema() + ), + default_timeout_seconds=( + builtin.default_timeout_seconds + if builtin is not None + else handler.default_timeout_seconds + ), + ) + ) + async with job_uow() as uow: + await uow.jobs.sync_types(rows) + + @classmethod + async def normalize_parameters( + cls, + job_type: JobTypeID, + parameters: dict[str, typing.Any], + uow: JobUnitOfWork, + ) -> dict[str, typing.Any]: + handler = cls._handlers.get(job_type) + if handler is not None: + return await handler.normalize_parameters(parameters, uow) + + persisted_type = await uow.jobs.get_type(job_type) + if persisted_type is None: + raise UnknownJobTypeError(f"Unknown Job type: {job_type}") + jsonschema.Draft202012Validator(persisted_type.parameters_schema).validate(parameters) + return parameters + + @classmethod + async def create( + cls, + job_type: JobTypeID, + parameters: dict[str, typing.Any], + timeout_seconds: int | None = None, + ) -> JobModel: + """Validate and persist one independent pending Job.""" + async with job_uow() as uow: + job = await cls.create_in_uow(job_type, parameters, timeout_seconds, uow=uow) + emit_event("job.submitted", {"inkcre.job.id": job.id or 0}) + return job + + @classmethod + async def create_in_uow( + cls, + job_type: JobTypeID, + parameters: dict[str, typing.Any], + timeout_seconds: int | None = None, + *, + uow: JobUnitOfWork, + ) -> JobModel: + normalized = await cls.normalize_parameters(job_type, parameters, uow) + persisted_type = await uow.jobs.get_type(job_type) + if persisted_type is None: + raise UnknownJobTypeError(f"Unknown Job type: {job_type}") + effective_timeout = ( + persisted_type.default_timeout_seconds if timeout_seconds is None else timeout_seconds + ) + if effective_timeout <= 0: + raise ValueError("Job timeout_seconds must be positive") + + with operation("job.submit") as observation: + span = observation.span + carrier = submission_context() + job = JobModel( + type=job_type, + parameters=normalized, + timeout_seconds=effective_timeout, + submission_traceparent=carrier["submission_traceparent"], + submission_tracestate=carrier["submission_tracestate"], + ) + job = await uow.jobs.create(job) + span.set_attribute("inkcre.job.id", job.id or 0) + # The composing use case still owns commit; this span only proves the flush. + span.set_attribute("inkcre.job.write_phase", "flushed") + return job + + @classmethod + async def _prepare( + cls, + job: JobModel, + ) -> tuple[type[JobHandler], pydantic.BaseModel] | None: + handler = cls._handlers.get(job.type) + if handler is None: + return None + parameters = handler.validate_parameters(job.parameters) + return (handler, parameters) if await handler.can_handle(parameters) else None + + @classmethod + async def get(cls, job_id: JobID) -> JobModel | None: + async with job_uow() as uow: + return await uow.jobs.get(job_id) + + @classmethod + async def get_type(cls, type_: JobTypeID) -> JobTypeModel | None: + async with job_uow() as uow: + return await uow.jobs.get_type(type_) + + @classmethod + async def list_types( + cls, *, limit: int | None = None, cursor: str | None = None + ) -> tuple[list[JobTypeModel], str | None]: + async with job_uow() as uow: + return await uow.jobs.list_types(limit=limit, cursor=cursor) + + @classmethod + async def list_jobs( + cls, + *, + limit: int = 20, + cursor: JobID | None = None, + type_: JobTypeID | None = None, + status: JobStatus | None = None, + ) -> tuple[list[JobModel], JobID | None]: + async with job_uow() as uow: + return await uow.jobs.list_jobs( + limit=limit, cursor=cursor, type_=type_, status=status + ) + + @classmethod + async def notify_worker(cls) -> None: + """Best-effort post-commit hint; scan failure cannot undo accepted work.""" + try: + await cls.check() + except Exception: + LOGGER.exception("Job discovery hint failed; periodic discovery remains active") + + @classmethod + async def abort(cls, job_id: JobID) -> JobModel | None: + """Close pending work or request the executing Peer to stop running work.""" + async with job_uow() as uow: + return await uow.jobs.abort(job_id) + + @classmethod + async def check_abort_requests(cls) -> None: + """Read this Peer's active abort requests in one batch.""" + if not current_core().active_jobs: + return + async with job_uow() as uow: + ids = await uow.jobs.abort_requests(tuple(current_core().active_jobs)) + for job_id in ids: + if job_id is None: + continue + execution = current_core().active_jobs.get(job_id) + if execution is not None: + execution.cancel() + + @classmethod + def start(cls) -> None: + current_core().accepting_jobs = True + + @classmethod + async def shutdown(cls) -> None: + """Stop admission and drain handlers before their Extension resources close.""" + current_core().accepting_jobs = False + scheduled = tuple(current_core().pending_jobs.values()) + for task in scheduled: + if not task.cancelling(): + task.cancel() + executions = tuple(current_core().active_jobs.values()) + for execution in executions: + execution.cancel() + await asyncio.gather( + *scheduled, *(item.task for item in executions), return_exceptions=True + ) + current_core().pending_jobs.clear() + + @classmethod + async def _claim(cls, job_id: JobID) -> JobModel | None: + if not current_core().accepting_jobs: + return None + async with job_uow() as uow: + return await uow.jobs.claim(job_id) + + @classmethod + async def _close(cls, job: JobModel, status: JobStatus) -> bool: + if not status.terminal: + raise ValueError("Job may close only to a terminal status") + execution = current_core().active_jobs.get(job.id) if job.id is not None else None + if execution is not None: + execution.closing = True + # Cleanup is independent of handler cancellation and cannot wait indefinitely. + async with asyncio.timeout(10): + async with job_uow() as uow: + closed = await uow.jobs.close(job, status) + if closed: + emit_event( + "job.closed", {"inkcre.job.id": job.id or 0, "inkcre.job.status": status.value} + ) + return closed + + @classmethod + async def run(cls, job_id: JobID) -> bool: + """Check local eligibility, atomically claim, then execute one Job.""" + if not current_core().accepting_jobs: + return False + candidate = await cls.get(job_id) + if candidate is None or candidate.status != JobStatus.PENDING: + return False + try: + prepared = await cls._prepare(candidate) + except pydantic.ValidationError as error: + # A broken persisted command must be visible as failed, not remain pending + # forever. Unknown/unavailable handlers still leave work for another Peer. + LOGGER.exception("Persisted Job parameters are invalid", extra={"job_id": job_id}) + claimed = await cls._claim(job_id) + if claimed is None: + return False + claimed.state = {**claimed.state, "error": str(error)} + await cls._close(claimed, JobStatus.FAILED) + return True + if prepared is None: + return False + + claimed = await cls._claim(job_id) + if claimed is None: + return False + handler, parameters = prepared + task = asyncio.current_task() + assert task is not None + current_core().active_jobs[job_id] = _Execution(task) + with operation( + "job.execute", + attributes={"inkcre.job.id": job_id}, + context=Context(), + links=submission_links(claimed.submission_traceparent, claimed.submission_tracestate), + ) as observation: + span = observation.span + emit_event("job.started", {"inkcre.job.id": job_id}) + try: + async with asyncio.timeout(claimed.timeout_seconds): + await handler.handle(claimed, parameters) + except asyncio.CancelledError: + LOGGER.info("Job execution aborted", extra={"job_id": job_id}) + observation.outcome = "cancelled" + span.set_attribute("inkcre.job.status", "aborted") + await cls._close(claimed, JobStatus.ABORTED) + except TimeoutError: + LOGGER.warning("Job execution timed out", extra={"job_id": job_id}) + observation.outcome = "error" + span.set_attribute("inkcre.job.status", "timed_out") + await cls._close(claimed, JobStatus.TIMED_OUT) + except Exception as error: + LOGGER.exception("Job execution failed", extra={"job_id": job_id}) + observation.outcome = "error" + span.set_attribute("error.type", type(error).__name__) + span.set_attribute("inkcre.job.status", "failed") + claimed.state = {**claimed.state, "error": str(error)} + await cls._close(claimed, JobStatus.FAILED) + else: + span.set_attribute("inkcre.job.status", "finished") + await cls._close(claimed, JobStatus.FINISHED) + finally: + current_core().active_jobs.pop(job_id, None) + return True + + @classmethod + async def expire_overdue(cls) -> int: + """Use database time to close abandoned overdue running Jobs.""" + async with job_uow() as uow: + return await uow.jobs.expire_overdue() + + @classmethod + async def check(cls) -> None: + """Schedule locally eligible pending Jobs and converge running timeouts.""" + if not current_core().accepting_jobs: + return + async with job_uow() as uow: + pending = await uow.jobs.pending() + + for job in pending: + if job.id is None or job.type not in cls._handlers: + continue + core = current_core() + if job.id in core.pending_jobs or job.id in core.active_jobs: + continue + task = asyncio.create_task(cls._run_traced(job.id), name=f"job.{job.id}") + core.pending_jobs[job.id] = task + task.add_done_callback( + lambda finished, job_id=job.id: core.pending_jobs.pop(job_id, None) + ) + await cls.expire_overdue() + + @classmethod + async def _run_traced(cls, job_id: JobID) -> bool: + token = TRACE_ID.set(f"job.{job_id}") + backend_token = ENABLE_LOG_BACKEND.set(True) + try: + return await cls.run(job_id) + except Exception: + LOGGER.exception("Scheduled Job failed", extra={"job_id": job_id}) + return False + finally: + TRACE_ID.reset(token) + ENABLE_LOG_BACKEND.reset(backend_token) diff --git a/inkcre_core/business/lexical_retrieval/__init__.py b/inkcre_core/business/lexical_retrieval/__init__.py new file mode 100644 index 00000000..56144cdf --- /dev/null +++ b/inkcre_core/business/lexical_retrieval/__init__.py @@ -0,0 +1,25 @@ +"""Exact Block-local lexical retrieval capability.""" + +from .main import ( + LEXICAL_RETRIEVAL_CAPABILITY, + LexicalRetrievalDelegationError, + LexicalRetrievalError, + LexicalRetrievalManager, +) +from .job import ( + LEXICAL_MAINTAIN_JOB_TYPE, + LEXICAL_REBUILD_JOB_TYPE, + LexicalMaintainJobHandler, + LexicalRebuildJobHandler, +) + +__all__ = [ + "LEXICAL_RETRIEVAL_CAPABILITY", + "LexicalRetrievalDelegationError", + "LexicalRetrievalError", + "LexicalRetrievalManager", + "LEXICAL_MAINTAIN_JOB_TYPE", + "LEXICAL_REBUILD_JOB_TYPE", + "LexicalMaintainJobHandler", + "LexicalRebuildJobHandler", +] diff --git a/inkcre_core/business/lexical_retrieval/job.py b/inkcre_core/business/lexical_retrieval/job.py new file mode 100644 index 00000000..f5747a85 --- /dev/null +++ b/inkcre_core/business/lexical_retrieval/job.py @@ -0,0 +1,56 @@ +"""Exact lexical projection maintenance commands hosted by Jobs.""" + +from inkcre_core.business.info_base.resolver import ResolverManager +from inkcre_core.business.job import JobHandler +from inkcre_core.schemas.job import JobModel +from inkcre_core.schemas.lexical_retrieval import LexicalMaintenanceJobParameters + +from .main import LexicalRetrievalManager + + +LEXICAL_MAINTAIN_JOB_TYPE = "core.feature_retrieval.lexical.maintain.v1" +LEXICAL_REBUILD_JOB_TYPE = "core.feature_retrieval.lexical.rebuild.v1" + + +class LexicalMaintainJobHandler( + JobHandler[LexicalMaintenanceJobParameters], + job_type=LEXICAL_MAINTAIN_JOB_TYPE, + description="Maintain missing or stale Block lexical projection records.", + parameters_model=LexicalMaintenanceJobParameters, + default_timeout_seconds=900, +): + @classmethod + async def can_handle(cls, parameters: LexicalMaintenanceJobParameters) -> bool: + del parameters + return bool(ResolverManager.RESOLVER_CLS) + + @classmethod + async def handle( + cls, + job: JobModel, + parameters: LexicalMaintenanceJobParameters, + ) -> None: + report = await LexicalRetrievalManager.maintain(parameters.options) + job.state = report.model_dump(mode="json") + + +class LexicalRebuildJobHandler( + JobHandler[LexicalMaintenanceJobParameters], + job_type=LEXICAL_REBUILD_JOB_TYPE, + description="Rebuild Block lexical records present before invocation.", + parameters_model=LexicalMaintenanceJobParameters, + default_timeout_seconds=1800, +): + @classmethod + async def can_handle(cls, parameters: LexicalMaintenanceJobParameters) -> bool: + del parameters + return bool(ResolverManager.RESOLVER_CLS) + + @classmethod + async def handle( + cls, + job: JobModel, + parameters: LexicalMaintenanceJobParameters, + ) -> None: + report = await LexicalRetrievalManager.rebuild(parameters.options) + job.state = report.model_dump(mode="json") diff --git a/inkcre_core/business/lexical_retrieval/main.py b/inkcre_core/business/lexical_retrieval/main.py new file mode 100644 index 00000000..ce5b8ca8 --- /dev/null +++ b/inkcre_core/business/lexical_retrieval/main.py @@ -0,0 +1,261 @@ +"""Block-local lexical projection maintenance and exact ranked retrieval.""" + +from __future__ import annotations + +from dataclasses import dataclass +import datetime +import logging +import typing + +import pydantic + +from inkcre_core.business.info_base.resolver import ( + ResolverManager, + UnknownResolverError, + UnsupportedResolverCapability, +) +from inkcre_core.business.peer import PeerManager +from inkcre_core.persistence.lexical_retrieval.uow import lexical_uow +from inkcre_core.schemas.info_base.block import BlockModel +from inkcre_core.schemas.lexical_retrieval import ( + LexicalEvidence, + LexicalMaintenanceDiagnostic, + LexicalMaintenanceOptions, + LexicalMaintenanceReport, + LexicalRetrievalMatch, + LexicalRetrievalRequest, + LexicalRetrievalResult, +) +from inkcre_core.schemas.peer import PeerProtocolRequest, PeerProtocolResponse, PeerRef + + +logger = logging.getLogger(__name__) + +LEXICAL_RETRIEVAL_CAPABILITY = "core.feature_retrieval.lexical.v1" +_EXCERPT_LIMIT = 280 + + +class LexicalRetrievalError(RuntimeError): + pass + + +class LexicalRetrievalDelegationError(LexicalRetrievalError): + pass + + +class _ProjectionUnavailable(Exception): + pass + + +@dataclass(frozen=True) +class _Projection: + block: int + label: str + text: str | None + + +@dataclass +class _ReportBuilder: + diagnostic_limit: int + indexed: int = 0 + unavailable: int = 0 + failed: int = 0 + diagnostics: list[LexicalMaintenanceDiagnostic] | None = None + + def __post_init__(self) -> None: + self.diagnostics = [] + + def record( + self, + block: int, + outcome: typing.Literal["unavailable", "failed"], + reason: str, + ) -> None: + if outcome == "unavailable": + self.unavailable += 1 + else: + self.failed += 1 + diagnostics = typing.cast(list[LexicalMaintenanceDiagnostic], self.diagnostics) + if len(diagnostics) < self.diagnostic_limit: + diagnostics.append( + LexicalMaintenanceDiagnostic(block=block, outcome=outcome, reason=reason) + ) + + def build(self) -> LexicalMaintenanceReport: + return LexicalMaintenanceReport( + indexed=self.indexed, + unavailable=self.unavailable, + failed=self.failed, + diagnostics=tuple(self.diagnostics or ()), + ) + + +class LexicalRetrievalManager: + """Sole owner of lexical records, maintenance, ranking, and delegation.""" + + @classmethod + async def retrieve( + cls, + query: str, + limit: int = 20, + *, + route_to_peer: PeerRef | None = None, + ) -> LexicalRetrievalResult: + request = LexicalRetrievalRequest(query=query, limit=limit) + if route_to_peer is None or route_to_peer == PeerManager.get_current_peer_ref(): + return await cls.retrieve_local(request.query, request.limit) + + payload = PeerProtocolRequest( + body=typing.cast(typing.Any, request.model_dump(mode="json")) + ) + result = await PeerManager.delegate( + LEXICAL_RETRIEVAL_CAPABILITY, + typing.cast(typing.Any, payload.model_dump(mode="json", exclude_unset=True)), + route_to_peer=route_to_peer, + ) + try: + response = PeerProtocolResponse.model_validate(result) + if response.status != 200 or "body" not in response.model_fields_set: + raise LexicalRetrievalDelegationError( + f"Lexical retrieval Peer returned HTTP {response.status}" + ) + return LexicalRetrievalResult.model_validate(response.body) + except pydantic.ValidationError as error: + raise LexicalRetrievalDelegationError( + "Lexical retrieval Peer returned an invalid response" + ) from error + + @classmethod + async def retrieve_local(cls, query: str, limit: int = 20) -> LexicalRetrievalResult: + request = LexicalRetrievalRequest(query=query, limit=limit) + normalized = request.query.strip() + async with lexical_uow() as repository: + rows = await repository.search(normalized, request.limit) + + return LexicalRetrievalResult( + matches=tuple( + LexicalRetrievalMatch( + block=block, + label=record.label, + excerpt=cls._excerpt(record.label, record.text, normalized), + evidence=typing.cast(LexicalEvidence, evidence_value), + rank=float(rank_value), + ) + for block, record, evidence_value, rank_value in rows + ) + ) + + @classmethod + async def maintain( + cls, + options: LexicalMaintenanceOptions | None = None, + ) -> LexicalMaintenanceReport: + return await cls._maintain( + options or LexicalMaintenanceOptions(), + rebuild_cutoff=None, + ) + + @classmethod + async def rebuild( + cls, + options: LexicalMaintenanceOptions | None = None, + ) -> LexicalMaintenanceReport: + async with lexical_uow() as repository: + cutoff = await repository.database_now() + return await cls._maintain( + options or LexicalMaintenanceOptions(), + rebuild_cutoff=cutoff, + ) + + @classmethod + async def _maintain( + cls, + options: LexicalMaintenanceOptions, + *, + rebuild_cutoff: datetime.datetime | None, + ) -> LexicalMaintenanceReport: + report = _ReportBuilder(options.diagnostic_limit) + cursor = 0 + processed = 0 + while processed < options.max_records: + async with lexical_uow() as repository: + blocks, next_cursor, exhausted = await repository.candidate_page( + cursor, + min(options.scan_page_size, options.max_records - processed), + rebuild_cutoff, + ) + cursor = next_cursor + projections: list[_Projection] = [] + for block in blocks: + block_id = typing.cast(int, block.id) + processed += 1 + try: + projections.append(await cls._project(block)) + except _ProjectionUnavailable as error: + report.record(block_id, "unavailable", str(error)) + except Exception as error: + logger.exception("Lexical projection failed", extra={"block": block_id}) + report.record(block_id, "failed", type(error).__name__) + if projections: + try: + async with lexical_uow() as repository: + await repository.upsert( + [ + { + "block": projection.block, + "label": projection.label, + "text": projection.text, + } + for projection in projections + ] + ) + except Exception as error: + logger.exception( + "Lexical projection batch upsert failed", + extra={"batch_size": len(projections)}, + ) + for projection in projections: + report.record(projection.block, "failed", type(error).__name__) + else: + report.indexed += len(projections) + if exhausted: + break + return report.build() + + @classmethod + async def _project(cls, block: BlockModel) -> _Projection: + block_id = block.id + if block_id is None: + raise RuntimeError("Persisted lexical candidate is missing its ID") + try: + resolver = ResolverManager.get(block) + label = await resolver.get_label() + text = await resolver.get_text(context="lexical", materialize_missing=True) + except UnknownResolverError as error: + raise _ProjectionUnavailable("unknown_resolver") from error + except UnsupportedResolverCapability as error: + raise _ProjectionUnavailable("unsupported_lexical_text") from error + normalized_label = label.strip() + normalized_text = text.strip() if text is not None and text.strip() else None + if not normalized_label: + raise _ProjectionUnavailable("empty_label") + return _Projection(block_id, normalized_label, normalized_text) + + @staticmethod + def _excerpt(label: str, text: str | None, query: str) -> str: + source = text or label + folded_source = source.casefold() + needle = query.casefold() + position = folded_source.find(needle) + if position < 0: + positions = [ + candidate + for term in query.split() + if (candidate := folded_source.find(term.casefold())) >= 0 + ] + position = min(positions, default=0) + start = max(0, position - _EXCERPT_LIMIT // 3) + end = min(len(source), start + _EXCERPT_LIMIT) + prefix = "…" if start else "" + suffix = "…" if end < len(source) else "" + return f"{prefix}{source[start:end]}{suffix}" diff --git a/inkcre_core/business/organization/__init__.py b/inkcre_core/business/organization/__init__.py new file mode 100644 index 00000000..2d7c161c --- /dev/null +++ b/inkcre_core/business/organization/__init__.py @@ -0,0 +1,92 @@ +"""Exact, extensible information-organization behaviors.""" + +from .bootstrap import register_core_organization_behaviors +from .contracts import ( + OrganizationAgentNotFoundError, + OrganizationBlockNotFoundError, + OrganizationBudgetExceededError, + OrganizationDelegationError, + OrganizationError, + OrganizationExecutionError, + OrganizationNotConfiguredError, +) +from .duplicate_assertion import ( + DUPLICATES_ASSERTION_RELATION, + DuplicateAssertionBehaviorResolver, +) +from .evidence_stance import ( + CHALLENGES_RELATION, + SUPPORTS_RELATION, + EvidenceStanceBehaviorResolver, +) +from .referent_anchoring import ( + HAS_MENTION_RELATION, + REFERS_TO_RELATION, + ExistingReferentAnchoringBehaviorResolver, +) +from .refinement import REFINES_RELATION, RefinementBehaviorResolver +from .rumination import ( + RUMINATION_CAPABILITY, + RUMINATION_CONFIG_KEY, + RUMINATION_CONFIG_SCHEMA, + RuminationBehaviorResolver, +) +from .supersession import ( + EDITED_RELATION, + SUPERSEDES_RELATION, + SupersessionBehaviorResolver, +) +from .synthesis import SYNTHESIS_RELATION, SynthesisBehaviorResolver +from .tools import ( + ANCHOR_EXISTING_REFERENT_TOOL, + CREATE_SYNTHESIS_TOOL, + DRAFT_GRAPH_TOOL, + GET_DRAFT_GRAPH_SCHEMA_TOOL, + RECORD_DUPLICATE_ASSERTION_TOOL, + RECORD_EVIDENCE_STANCE_TOOL, + RECORD_ORGANIZATION_CANDIDATE_TOOL, + RECORD_REFINEMENT_TOOL, + RECORD_SUPERSESSION_TOOL, + SUBMIT_GRAPH_TOOL, +) + + +__all__ = [ + "ANCHOR_EXISTING_REFERENT_TOOL", + "CHALLENGES_RELATION", + "CREATE_SYNTHESIS_TOOL", + "DRAFT_GRAPH_TOOL", + "DUPLICATES_ASSERTION_RELATION", + "DuplicateAssertionBehaviorResolver", + "EDITED_RELATION", + "EvidenceStanceBehaviorResolver", + "ExistingReferentAnchoringBehaviorResolver", + "GET_DRAFT_GRAPH_SCHEMA_TOOL", + "HAS_MENTION_RELATION", + "OrganizationAgentNotFoundError", + "OrganizationBlockNotFoundError", + "OrganizationBudgetExceededError", + "OrganizationDelegationError", + "OrganizationError", + "OrganizationExecutionError", + "OrganizationNotConfiguredError", + "RECORD_DUPLICATE_ASSERTION_TOOL", + "RECORD_EVIDENCE_STANCE_TOOL", + "RECORD_ORGANIZATION_CANDIDATE_TOOL", + "RECORD_REFINEMENT_TOOL", + "RECORD_SUPERSESSION_TOOL", + "REFERS_TO_RELATION", + "REFINES_RELATION", + "RUMINATION_CAPABILITY", + "RUMINATION_CONFIG_KEY", + "RUMINATION_CONFIG_SCHEMA", + "RuminationBehaviorResolver", + "SUBMIT_GRAPH_TOOL", + "SUPERSEDES_RELATION", + "SUPPORTS_RELATION", + "SYNTHESIS_RELATION", + "SupersessionBehaviorResolver", + "SynthesisBehaviorResolver", + "RefinementBehaviorResolver", + "register_core_organization_behaviors", +] diff --git a/inkcre_core/business/organization/_shared.py b/inkcre_core/business/organization/_shared.py new file mode 100644 index 00000000..6499e80a --- /dev/null +++ b/inkcre_core/business/organization/_shared.py @@ -0,0 +1,365 @@ +"""Small shared mechanics for exact Organization behaviors.""" + +from __future__ import annotations + +from contextlib import contextmanager +import json +import logging +import typing + +import pydantic + +from inkcre_core.business.agent import AgentManager, AgentNotFoundError, TurnTermination +from inkcre_core.business.deployment_config import DeploymentConfigService +from inkcre_core.business.info_base.resolver import ( + Resolver, + ResolverManager, + UnknownResolverError, + UnsupportedResolverCapability, +) +from inkcre_core.persistence.info_base.uow import GraphUnitOfWork, graph_uow +from inkcre_core.schemas.ai import TextContentPart, UserMessage +from inkcre_core.schemas.info_base.block import BlockForm, BlockID, BlockModel, ResolverType +from inkcre_core.schemas.info_base.relation import RelationID, RelationModel +from inkcre_core.schemas.organization_behavior import ( + CandidateWriteResult, + RelationWriteResult, +) + +from .contracts import ( + OrganizationAgentNotFoundError, + OrganizationBlockNotFoundError, + OrganizationBudgetExceededError, + OrganizationExecutionError, + OrganizationNotConfiguredError, +) + + +CANDIDATE_RELATION = "candidate for" +_CONTEXT_RELATION_LIMIT = 20 + + +def behavior_resolver_classes() -> tuple[type[Resolver], ...]: + classes: list[type[Resolver]] = [] + for resolver_cls in ResolverManager.RESOLVER_CLS.values(): + if ( + issubclass(resolver_cls, Resolver) + and isinstance(getattr(resolver_cls, "organization_description", None), str) + and callable(getattr(resolver_cls, "record_candidate", None)) + ): + classes.append(resolver_cls) + return tuple(sorted(classes, key=lambda resolver_cls: resolver_cls.__rsotype__)) + + +def get_behavior_resolver(resolver: ResolverType) -> type[Resolver] | None: + return next( + ( + resolver_cls + for resolver_cls in behavior_resolver_classes() + if resolver_cls.__rsotype__ == resolver + ), + None, + ) + + +async def get_or_create_descriptor( + behavior: type[Resolver], + uow: GraphUnitOfWork, +) -> BlockModel: + form = BlockForm(resolver=behavior.__rsotype__, content="") + existing = await ResolverManager.get(BlockModel.model_validate(form)).get_existing_async( + uow.blocks + ) + return existing if existing is not None else await uow.blocks.create(form) + + +async def record_candidate( + behavior: type[Resolver], + block_id: BlockID, +) -> CandidateWriteResult: + async with graph_uow() as uow: + if (await uow.blocks.get(block_id)) is None: + raise OrganizationBlockNotFoundError(f"Block {block_id} does not exist") + descriptor = await get_or_create_descriptor(behavior, uow) + descriptor_id = _block_id(descriptor) + if block_id == descriptor_id: + raise ValueError("An Organization behavior cannot be its own candidate") + relation, created = await fetchsert_relation( + block_id, + descriptor_id, + CANDIDATE_RELATION, + uow, + ) + return CandidateWriteResult( + descriptor_block_id=descriptor_id, + relation_id=_relation_id(relation), + created=created, + ) + + +async def candidate_seed_ids( + behavior: type[Resolver], + limit: int, +) -> tuple[BlockID, ...]: + if limit <= 0: + return () + async with graph_uow() as uow: + descriptor = await get_or_create_descriptor(behavior, uow) + descriptor_id = _block_id(descriptor) + return await uow.relations.random_incoming_sources( + descriptor_id, CANDIDATE_RELATION, limit + ) + + +async def recent_block_ids(limit: int) -> tuple[BlockID, ...]: + if limit <= 0: + return () + behavior_types = tuple( + resolver_cls.__rsotype__ for resolver_cls in behavior_resolver_classes() + ) + async with graph_uow() as uow: + return await uow.blocks.select_ids( + limit, exclude_resolvers=behavior_types, random_order=False + ) + + +async def random_block_ids(limit: int) -> tuple[BlockID, ...]: + if limit <= 0: + return () + behavior_types = tuple( + resolver_cls.__rsotype__ for resolver_cls in behavior_resolver_classes() + ) + async with graph_uow() as uow: + return await uow.blocks.select_ids( + limit, exclude_resolvers=behavior_types, random_order=True + ) + + +async def recent_relation_endpoint_ids( + limit: int, + *, + contents: typing.Collection[str] = (), +) -> tuple[BlockID, ...]: + if limit <= 0: + return () + async with graph_uow() as uow: + relations = await uow.relations.recent(limit, contents=contents) + return tuple( + dict.fromkeys( + endpoint for relation in relations for endpoint in (relation.from_, relation.to_) + ) + ) + + +def merge_seed_categories( + max_seeds: int, + *categories: typing.Collection[BlockID], +) -> tuple[BlockID, ...]: + """Reserve one position per non-empty category, then fill in priority order.""" + result: list[BlockID] = [] + normalized = [tuple(dict.fromkeys(category)) for category in categories] + for category in normalized: + if category and category[0] not in result: + result.append(category[0]) + if len(result) == max_seeds: + return tuple(result) + for category in normalized: + for block_id in category[1:]: + if block_id not in result: + result.append(block_id) + if len(result) == max_seeds: + return tuple(result) + return tuple(result) + + +async def configured_agent_available( + config_key: str, + config_type: type[pydantic.BaseModel], +) -> bool: + config = await DeploymentConfigService.get(config_key) + if config is None: + return False + if not isinstance(config, config_type): + raise TypeError(f"Organization config {config_key!r} returned the wrong model") + return await AgentManager.can_execute(typing.cast(typing.Any, config).agent, "text") + + +@contextmanager +def continue_after_seed_failure( + logger: logging.Logger, + behavior: ResolverType, + seed_block_id: BlockID, +) -> typing.Generator[None, None, None]: + """Recover only candidate-local failures in an automatic attempt.""" + try: + yield + except (OrganizationBlockNotFoundError, OrganizationBudgetExceededError) as error: + logger.warning( + "organization.seed.considered", + extra={ + "behavior": behavior, + "seed_block_ids": (seed_block_id,), + "outcome": "recoverable_failure", + "reason": ( + "seed_missing" + if isinstance(error, OrganizationBlockNotFoundError) + else "model_call_limit" + ), + }, + ) + + +async def run_configured_agent( + config_key: str, + config_type: type[pydantic.BaseModel], + message: UserMessage, +) -> None: + config = await DeploymentConfigService.get(config_key) + if config is None: + raise OrganizationNotConfiguredError( + f"Organization behavior {config_key!r} is not configured" + ) + if not isinstance(config, config_type): + raise TypeError(f"Organization config {config_key!r} returned the wrong model") + agent_id = typing.cast(typing.Any, config).agent + try: + thread = await AgentManager.run(agent_id, message) + except AgentNotFoundError as error: + raise OrganizationAgentNotFoundError( + f"Configured Organization Agent {agent_id} does not exist" + ) from error + turn = thread.current_turn + if turn is None: # pragma: no cover - AgentManager.run invariant + raise OrganizationExecutionError("Organization Agent did not start a Turn") + outcome = await turn + if outcome == TurnTermination.MAX_MODEL_CALLS: + raise OrganizationBudgetExceededError( + "Organization Agent exceeded its per-Turn model-call budget" + ) + + +async def build_seed_message( + request: str, + judgment_contract: tuple[str, ...], + seed_id: BlockID, +) -> UserMessage | None: + """Resolve one bounded seed neighborhood without holding a DB transaction.""" + async with graph_uow() as uow: + block = await uow.blocks.get(seed_id) + if block is None: + raise OrganizationBlockNotFoundError(f"Block {seed_id} does not exist") + relations = tuple( + sorted( + (await uow.relations.get(seed_id)), + key=lambda relation: relation.id or 0, + )[-_CONTEXT_RELATION_LIMIT:] + ) + neighbor_ids = { + relation.to_ if relation.from_ == seed_id else relation.from_ + for relation in relations + } + neighbors = { + neighbor.id: neighbor + for neighbor in (await uow.blocks.get_many(neighbor_ids)) + if neighbor.id is not None + } + + try: + resolver = ResolverManager.get(block) + text = await resolver.get_text(materialize_missing=False) + label = await resolver.get_label() + except (UnknownResolverError, UnsupportedResolverCapability): + return None + if text is None or not text.strip(): + return None + + relation_context: list[dict[str, typing.Any]] = [] + for relation in relations: + other_id = relation.to_ if relation.from_ == seed_id else relation.from_ + other = neighbors.get(other_id) + other_label = None + if other is not None: + try: + other_label = await ResolverManager.get(other).get_label() + except (UnknownResolverError, UnsupportedResolverCapability): + pass + relation_context.append( + { + "id": relation.id, + "direction": "outgoing" if relation.from_ == seed_id else "incoming", + "content": relation.content, + "other_block": { + "id": other_id, + "resolver": other.resolver if other is not None else None, + "label": other_label, + }, + } + ) + context = { + "request": request, + "judgment_contract": judgment_contract, + "seed_block": { + "id": seed_id, + "resolver": block.resolver, + "label": label, + "text": text, + }, + "direct_relations": relation_context, + "exploration": ( + "The seed and its direct relations are only a starting point. Use the declared " + "retrieval, Resolver, and graph tools when more evidence is needed. Persist only " + "through the exact behavior tool, or cautiously mark a different behavior candidate." + ), + } + return UserMessage( + content=( + TextContentPart( + text=json.dumps( + context, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ) + ), + ) + ) + + +async def fetchsert_relation( + from_: BlockID, + to_: BlockID, + content: str, + uow: GraphUnitOfWork, +) -> tuple[RelationModel, bool]: + proposed = RelationModel(from_=from_, to_=to_, content=content) + relation = await uow.relations.fetchsert(proposed) + return relation, relation is proposed + + +def relation_result(relation: RelationModel, created: bool) -> RelationWriteResult: + return RelationWriteResult(relation_id=_relation_id(relation), created=created) + + +async def require_distinct_blocks( + left: BlockID, + right: BlockID, + uow: GraphUnitOfWork, +) -> None: + if left == right: + raise ValueError("Organization relation endpoints must be different") + found = {block.id for block in (await uow.blocks.get_many((left, right)))} + missing = tuple(block_id for block_id in (left, right) if block_id not in found) + if missing: + raise OrganizationBlockNotFoundError(f"Blocks do not exist: {missing!r}") + + +def _block_id(block: BlockModel) -> BlockID: + if block.id is None: # pragma: no cover - persisted Block invariant + raise RuntimeError("Persisted Block has no ID") + return block.id + + +def _relation_id(relation: RelationModel) -> RelationID: + if relation.id is None: # pragma: no cover - persisted Relation invariant + raise RuntimeError("Persisted Relation has no ID") + return relation.id diff --git a/inkcre_core/business/organization/bootstrap.py b/inkcre_core/business/organization/bootstrap.py new file mode 100644 index 00000000..d769a4ef --- /dev/null +++ b/inkcre_core/business/organization/bootstrap.py @@ -0,0 +1,24 @@ +"""Explicit loading of core Organization Behavior Resolvers.""" + +from inkcre_core.business.info_base.resolver import ResolverManager + + +def register_core_organization_behaviors() -> None: + from .duplicate_assertion import DuplicateAssertionBehaviorResolver + from .evidence_stance import EvidenceStanceBehaviorResolver + from .referent_anchoring import ExistingReferentAnchoringBehaviorResolver + from .refinement import RefinementBehaviorResolver + from .rumination import RuminationBehaviorResolver + from .supersession import SupersessionBehaviorResolver + from .synthesis import SynthesisBehaviorResolver + + for resolver_class in ( + RuminationBehaviorResolver, + SupersessionBehaviorResolver, + RefinementBehaviorResolver, + EvidenceStanceBehaviorResolver, + SynthesisBehaviorResolver, + ExistingReferentAnchoringBehaviorResolver, + DuplicateAssertionBehaviorResolver, + ): + ResolverManager.register_resolver(resolver_class) diff --git a/inkcre_core/business/organization/contracts.py b/inkcre_core/business/organization/contracts.py new file mode 100644 index 00000000..a395a533 --- /dev/null +++ b/inkcre_core/business/organization/contracts.py @@ -0,0 +1,29 @@ +"""Stable failures at the Organization capability boundary.""" + + +class OrganizationError(RuntimeError): + """Base failure at the Organization capability boundary.""" + + +class OrganizationBlockNotFoundError(OrganizationError): + pass + + +class OrganizationNotConfiguredError(OrganizationError): + pass + + +class OrganizationAgentNotFoundError(OrganizationError): + pass + + +class OrganizationExecutionError(OrganizationError): + pass + + +class OrganizationBudgetExceededError(OrganizationExecutionError): + """One Agent Turn ended at its configured model-call limit.""" + + +class OrganizationDelegationError(OrganizationError): + pass diff --git a/inkcre_core/business/organization/duplicate_assertion.py b/inkcre_core/business/organization/duplicate_assertion.py new file mode 100644 index 00000000..599d1f37 --- /dev/null +++ b/inkcre_core/business/organization/duplicate_assertion.py @@ -0,0 +1,160 @@ +"""Provenance-aware duplicate assertion behavior.""" + +from __future__ import annotations + +import typing + + +from inkcre_core.business.deployment_config import DeploymentConfigManager +from inkcre_core.business.info_base.resolver import Resolver +from inkcre_core.persistence.info_base.uow import graph_uow +from inkcre_core.libs.obsrv.main import get_logger +from inkcre_core.schemas.info_base.block import BlockID +from inkcre_core.schemas.organization_behavior import ( + BehaviorAgentConfig, + CandidateWriteResult, + RelationWriteResult, +) + +from ._shared import ( + build_seed_message, + candidate_seed_ids, + configured_agent_available, + continue_after_seed_failure, + fetchsert_relation, + merge_seed_categories, + random_block_ids, + recent_block_ids, + record_candidate, + relation_result, + require_distinct_blocks, + run_configured_agent, +) + + +LOGGER = get_logger().getChild(__name__) + +DUPLICATES_ASSERTION_RELATION = "duplicates assertion" +DUPLICATE_ASSERTION_BEHAVIOR = "core.organization.behavior.duplicate-assertion.v1" +DUPLICATE_ASSERTION_CONFIG_KEY = "core.organization.duplicate_assertion" +DUPLICATE_ASSERTION_CONFIG_SCHEMA = "core.organization.duplicate_assertion.config.v1" + +DeploymentConfigManager.register_schema( + DUPLICATE_ASSERTION_CONFIG_SCHEMA, + BehaviorAgentConfig, + keys=(DUPLICATE_ASSERTION_CONFIG_KEY,), +) + + +class DuplicateAssertionBehaviorResolver( + Resolver[str, str], + rso_type=DUPLICATE_ASSERTION_BEHAVIOR, +): + organization_description = ( + "Relate whole-Block assertions copied from the same provenance occurrence." + ) + judgment_contract = ( + "Both Blocks completely and independently address the compared assertion.", + "Referent, predicate, polarity, force, units, and material qualifiers match.", + "Applicable scope, time, version, environment, and attribution are compatible.", + "Both assertions ultimately derive from the same observable provenance occurrence.", + "Neither Block adds independent evidence, reasoning, or authoritative decision.", + "No material asymmetric information gain is hidden by the relation.", + "The relation prevents evidence multiplication or restores useful provenance paths.", + ) + + async def get_text( + self, + *, + context: typing.Literal["default", "lexical"] = "default", + refresh: bool = False, + materialize_missing: bool = True, + ) -> str: + del context, refresh, materialize_missing + return self.organization_description + + async def get_label(self, *, refresh: bool = False) -> str: + del refresh + return "organization behavior: duplicate assertion" + + @classmethod + async def record_candidate( + cls, + block_id: BlockID, + ) -> CandidateWriteResult: + return await record_candidate(cls, block_id) + + @classmethod + async def record_duplicate_assertion( + cls, + left_block_id: BlockID, + right_block_id: BlockID, + ) -> RelationWriteResult: + async with graph_uow() as uow: + await require_distinct_blocks(left_block_id, right_block_id, uow) + from_, to_ = sorted((left_block_id, right_block_id)) + relation, created = await fetchsert_relation( + from_, + to_, + DUPLICATES_ASSERTION_RELATION, + uow, + ) + return relation_result(relation, created) + + @classmethod + async def can_run_automatic(cls) -> bool: + return await configured_agent_available( + DUPLICATE_ASSERTION_CONFIG_KEY, + BehaviorAgentConfig, + ) + + @classmethod + async def run_automatic(cls, max_seeds: int) -> None: + candidates = await candidate_seed_ids(cls, max_seeds) + recent = await recent_block_ids(max_seeds) + random = await random_block_ids(max_seeds) + seeds = merge_seed_categories(max_seeds, candidates, recent, random) + LOGGER.info( + "organization.seeds.selected", + extra={ + "behavior": cls.__rsotype__, + "max_seeds": max_seeds, + "candidate_count": len(candidates), + "recent_count": len(recent), + "random_count": len(random), + "seed_block_ids": seeds, + }, + ) + for seed in seeds: + with continue_after_seed_failure(LOGGER, cls.__rsotype__, seed): + message = await build_seed_message( + "Record only whole-Block duplicate assertions " + "from the same provenance occurrence.", + cls.judgment_contract, + seed, + ) + if message is None: + LOGGER.info( + "organization.seed.considered", + extra={ + "behavior": cls.__rsotype__, + "seed_block_ids": (seed,), + "outcome": "unresolved", + "reason": "text_unavailable", + }, + ) + continue + await run_configured_agent( + DUPLICATE_ASSERTION_CONFIG_KEY, + BehaviorAgentConfig, + message, + ) + LOGGER.info( + "organization.seed.considered", + extra={ + "behavior": cls.__rsotype__, + "seed_block_ids": (seed,), + "outcome": "considered", + "reason": "agent_completed", + }, + ) diff --git a/inkcre_core/business/organization/evidence_stance.py b/inkcre_core/business/organization/evidence_stance.py new file mode 100644 index 00000000..03e5b555 --- /dev/null +++ b/inkcre_core/business/organization/evidence_stance.py @@ -0,0 +1,169 @@ +"""Provenance-preserving evidence stance relation behavior.""" + +from __future__ import annotations + +import typing + + +from inkcre_core.business.deployment_config import DeploymentConfigManager +from inkcre_core.business.info_base.resolver import Resolver +from inkcre_core.persistence.info_base.uow import graph_uow +from inkcre_core.libs.obsrv.main import get_logger +from inkcre_core.schemas.info_base.block import BlockID +from inkcre_core.schemas.organization_behavior import ( + BehaviorAgentConfig, + CandidateWriteResult, + RelationWriteResult, +) + +from ._shared import ( + build_seed_message, + candidate_seed_ids, + configured_agent_available, + continue_after_seed_failure, + fetchsert_relation, + merge_seed_categories, + random_block_ids, + recent_block_ids, + recent_relation_endpoint_ids, + record_candidate, + relation_result, + require_distinct_blocks, + run_configured_agent, +) + + +LOGGER = get_logger().getChild(__name__) + +SUPPORTS_RELATION = "supports" +CHALLENGES_RELATION = "challenges" +EVIDENCE_STANCE_BEHAVIOR = "core.organization.behavior.evidence-stance.v1" +EVIDENCE_STANCE_CONFIG_KEY = "core.organization.evidence_stance" +EVIDENCE_STANCE_CONFIG_SCHEMA = "core.organization.evidence_stance.config.v1" + +DeploymentConfigManager.register_schema( + EVIDENCE_STANCE_CONFIG_SCHEMA, + BehaviorAgentConfig, + keys=(EVIDENCE_STANCE_CONFIG_KEY,), +) + + +class EvidenceStanceBehaviorResolver( + Resolver[str, str], + rso_type=EVIDENCE_STANCE_BEHAVIOR, +): + organization_description = ( + "Relate attributable evidence that supports or challenges an assertion." + ) + judgment_contract = ( + "Evidence and assertion are complete addressable information units.", + "The source is evidence and the target is an evaluable assertion.", + "Their proposition and applicable scope are comparable.", + "The evidence contributes reasons beyond establishing source fidelity; " + "source authority alone is insufficient.", + "Evidence provenance and speaker attribution remain recoverable.", + "The stance is unambiguously support or challenge for the whole assertion.", + "Duplicate, refinement, replacement, or topical proximity alone is insufficient.", + ) + + async def get_text( + self, + *, + context: typing.Literal["default", "lexical"] = "default", + refresh: bool = False, + materialize_missing: bool = True, + ) -> str: + del context, refresh, materialize_missing + return self.organization_description + + async def get_label(self, *, refresh: bool = False) -> str: + del refresh + return "organization behavior: evidence stance" + + @classmethod + async def record_candidate( + cls, + block_id: BlockID, + ) -> CandidateWriteResult: + return await record_candidate(cls, block_id) + + @classmethod + async def record_evidence_stance( + cls, + evidence_block_id: BlockID, + assertion_block_id: BlockID, + stance: typing.Literal["supports", "challenges"], + ) -> RelationWriteResult: + async with graph_uow() as uow: + await require_distinct_blocks(evidence_block_id, assertion_block_id, uow) + opposite = CHALLENGES_RELATION if stance == SUPPORTS_RELATION else SUPPORTS_RELATION + existing_opposite = await uow.relations.get_outgoing_many( + (evidence_block_id,), to_ids=(assertion_block_id,), content=opposite + ) + if existing_opposite: + raise ValueError("The evidence/assertion pair already has the opposite stance") + relation, created = await fetchsert_relation( + evidence_block_id, + assertion_block_id, + stance, + uow, + ) + return relation_result(relation, created) + + @classmethod + async def can_run_automatic(cls) -> bool: + return await configured_agent_available(EVIDENCE_STANCE_CONFIG_KEY, BehaviorAgentConfig) + + @classmethod + async def run_automatic(cls, max_seeds: int) -> None: + candidates = await candidate_seed_ids(cls, max_seeds) + strong = merge_seed_categories( + max_seeds, + await recent_relation_endpoint_ids(max_seeds), + await recent_block_ids(max_seeds), + ) + random = await random_block_ids(max_seeds) + seeds = merge_seed_categories(max_seeds, candidates, strong, random) + LOGGER.info( + "organization.seeds.selected", + extra={ + "behavior": cls.__rsotype__, + "max_seeds": max_seeds, + "candidate_count": len(candidates), + "strong_count": len(strong), + "random_count": len(random), + "seed_block_ids": seeds, + }, + ) + for seed in seeds: + with continue_after_seed_failure(LOGGER, cls.__rsotype__, seed): + message = await build_seed_message( + "Determine only attributable evidence support or challenge relations.", + cls.judgment_contract, + seed, + ) + if message is None: + LOGGER.info( + "organization.seed.considered", + extra={ + "behavior": cls.__rsotype__, + "seed_block_ids": (seed,), + "outcome": "unresolved", + "reason": "text_unavailable", + }, + ) + continue + await run_configured_agent( + EVIDENCE_STANCE_CONFIG_KEY, + BehaviorAgentConfig, + message, + ) + LOGGER.info( + "organization.seed.considered", + extra={ + "behavior": cls.__rsotype__, + "seed_block_ids": (seed,), + "outcome": "considered", + "reason": "agent_completed", + }, + ) diff --git a/inkcre_core/business/organization/jobs.py b/inkcre_core/business/organization/jobs.py new file mode 100644 index 00000000..e57b02f0 --- /dev/null +++ b/inkcre_core/business/organization/jobs.py @@ -0,0 +1,197 @@ +"""Independent automatic Organization routes and explicit rumination Jobs.""" + +from inkcre_core.business.job import JobHandler +from inkcre_core.schemas.job import JobModel +from inkcre_core.schemas.organization import RuminationRequest +from inkcre_core.schemas.organization_behavior import AutomaticOrganizationJobParameters + +from .duplicate_assertion import DuplicateAssertionBehaviorResolver +from .evidence_stance import EvidenceStanceBehaviorResolver +from .referent_anchoring import ExistingReferentAnchoringBehaviorResolver +from .refinement import RefinementBehaviorResolver +from .rumination import RuminationBehaviorResolver +from .supersession import SupersessionBehaviorResolver +from .synthesis import SynthesisBehaviorResolver + + +RUMINATION_JOB_TYPE = "core.organization.rumination.automatic.v1" +EXPLICIT_RUMINATION_JOB_TYPE = "core.organization.rumination.explicit.v1" +SUPERSESSION_JOB_TYPE = "core.organization.supersession.automatic.v1" +REFINEMENT_JOB_TYPE = "core.organization.refinement.automatic.v1" +EVIDENCE_STANCE_JOB_TYPE = "core.organization.evidence-stance.automatic.v1" +SYNTHESIS_JOB_TYPE = "core.organization.synthesis.automatic.v1" +REFERENT_ANCHORING_JOB_TYPE = "core.organization.existing-referent-anchoring.automatic.v1" +DUPLICATE_ASSERTION_JOB_TYPE = "core.organization.duplicate-assertion.automatic.v1" + + +class ExplicitRuminationJobHandler( + JobHandler[RuminationRequest], + job_type=EXPLICIT_RUMINATION_JOB_TYPE, + description="Reconsider one explicitly selected Block through rumination.", + parameters_model=RuminationRequest, + default_timeout_seconds=1800, +): + @classmethod + async def can_handle(cls, parameters: RuminationRequest) -> bool: + del parameters + return await RuminationBehaviorResolver.can_run_automatic() + + @classmethod + async def handle(cls, job: JobModel, parameters: RuminationRequest) -> None: + del job + # A claimed Job executes here; it must not delegate another execution. + await RuminationBehaviorResolver.ruminate_local(parameters.block) + + +class RuminationJobHandler( + JobHandler[AutomaticOrganizationJobParameters], + job_type=RUMINATION_JOB_TYPE, + description="Automatically reconsider bounded information seeds through rumination.", + parameters_model=AutomaticOrganizationJobParameters, + default_timeout_seconds=1800, +): + @classmethod + async def can_handle(cls, parameters: AutomaticOrganizationJobParameters) -> bool: + del parameters + return await RuminationBehaviorResolver.can_run_automatic() + + @classmethod + async def handle( + cls, + job: JobModel, + parameters: AutomaticOrganizationJobParameters, + ) -> None: + del job + await RuminationBehaviorResolver.run_automatic(parameters.max_seeds) + + +class SupersessionJobHandler( + JobHandler[AutomaticOrganizationJobParameters], + job_type=SUPERSESSION_JOB_TYPE, + description="Automatically judge bounded scoped-supersession candidates.", + parameters_model=AutomaticOrganizationJobParameters, + default_timeout_seconds=1800, +): + @classmethod + async def can_handle(cls, parameters: AutomaticOrganizationJobParameters) -> bool: + del parameters + return await SupersessionBehaviorResolver.can_run_automatic() + + @classmethod + async def handle( + cls, + job: JobModel, + parameters: AutomaticOrganizationJobParameters, + ) -> None: + del job + await SupersessionBehaviorResolver.run_automatic(parameters.max_seeds) + + +class RefinementJobHandler( + JobHandler[AutomaticOrganizationJobParameters], + job_type=REFINEMENT_JOB_TYPE, + description="Automatically judge bounded non-dominating refinement candidates.", + parameters_model=AutomaticOrganizationJobParameters, + default_timeout_seconds=1800, +): + @classmethod + async def can_handle(cls, parameters: AutomaticOrganizationJobParameters) -> bool: + del parameters + return await RefinementBehaviorResolver.can_run_automatic() + + @classmethod + async def handle( + cls, + job: JobModel, + parameters: AutomaticOrganizationJobParameters, + ) -> None: + del job + await RefinementBehaviorResolver.run_automatic(parameters.max_seeds) + + +class EvidenceStanceJobHandler( + JobHandler[AutomaticOrganizationJobParameters], + job_type=EVIDENCE_STANCE_JOB_TYPE, + description="Automatically judge bounded attributable evidence stances.", + parameters_model=AutomaticOrganizationJobParameters, + default_timeout_seconds=1800, +): + @classmethod + async def can_handle(cls, parameters: AutomaticOrganizationJobParameters) -> bool: + del parameters + return await EvidenceStanceBehaviorResolver.can_run_automatic() + + @classmethod + async def handle( + cls, + job: JobModel, + parameters: AutomaticOrganizationJobParameters, + ) -> None: + del job + await EvidenceStanceBehaviorResolver.run_automatic(parameters.max_seeds) + + +class SynthesisJobHandler( + JobHandler[AutomaticOrganizationJobParameters], + job_type=SYNTHESIS_JOB_TYPE, + description="Automatically create bounded provenance-preserving syntheses.", + parameters_model=AutomaticOrganizationJobParameters, + default_timeout_seconds=1800, +): + @classmethod + async def can_handle(cls, parameters: AutomaticOrganizationJobParameters) -> bool: + del parameters + return await SynthesisBehaviorResolver.can_run_automatic() + + @classmethod + async def handle( + cls, + job: JobModel, + parameters: AutomaticOrganizationJobParameters, + ) -> None: + del job + await SynthesisBehaviorResolver.run_automatic(parameters.max_seeds) + + +class ExistingReferentAnchoringJobHandler( + JobHandler[AutomaticOrganizationJobParameters], + job_type=REFERENT_ANCHORING_JOB_TYPE, + description="Automatically anchor bounded mentions to existing referents.", + parameters_model=AutomaticOrganizationJobParameters, + default_timeout_seconds=1800, +): + @classmethod + async def can_handle(cls, parameters: AutomaticOrganizationJobParameters) -> bool: + del parameters + return await ExistingReferentAnchoringBehaviorResolver.can_run_automatic() + + @classmethod + async def handle( + cls, + job: JobModel, + parameters: AutomaticOrganizationJobParameters, + ) -> None: + del job + await ExistingReferentAnchoringBehaviorResolver.run_automatic(parameters.max_seeds) + + +class DuplicateAssertionJobHandler( + JobHandler[AutomaticOrganizationJobParameters], + job_type=DUPLICATE_ASSERTION_JOB_TYPE, + description="Automatically judge bounded provenance-aware duplicate assertions.", + parameters_model=AutomaticOrganizationJobParameters, + default_timeout_seconds=1800, +): + @classmethod + async def can_handle(cls, parameters: AutomaticOrganizationJobParameters) -> bool: + del parameters + return await DuplicateAssertionBehaviorResolver.can_run_automatic() + + @classmethod + async def handle( + cls, + job: JobModel, + parameters: AutomaticOrganizationJobParameters, + ) -> None: + del job + await DuplicateAssertionBehaviorResolver.run_automatic(parameters.max_seeds) diff --git a/inkcre_core/business/organization/referent_anchoring.py b/inkcre_core/business/organization/referent_anchoring.py new file mode 100644 index 00000000..a3c33a66 --- /dev/null +++ b/inkcre_core/business/organization/referent_anchoring.py @@ -0,0 +1,273 @@ +"""Existing-referent anchoring through occurrence-local selected text.""" + +from __future__ import annotations + +import typing + + +from inkcre_core.business.deployment_config import DeploymentConfigManager +from inkcre_core.business.info_base.resolver import Resolver +from inkcre_core.persistence.info_base.uow import GraphUnitOfWork, graph_uow +from inkcre_core.libs.obsrv.main import get_logger +from inkcre_core.schemas.info_base.block import BlockForm, BlockID +from inkcre_core.schemas.info_base.relation import RelationModel +from inkcre_core.schemas.organization_behavior import ( + BehaviorAgentConfig, + CandidateWriteResult, + ExistingReferentAnchorProposal, + ExistingReferentAnchorResult, +) + +from ._shared import ( + build_seed_message, + candidate_seed_ids, + configured_agent_available, + continue_after_seed_failure, + fetchsert_relation, + merge_seed_categories, + random_block_ids, + recent_block_ids, + record_candidate, + relation_result, + require_distinct_blocks, + run_configured_agent, +) + + +LOGGER = get_logger().getChild(__name__) + +HAS_MENTION_RELATION = "has mention" +REFERS_TO_RELATION = "refers to" +REFERENT_ANCHORING_BEHAVIOR = "core.organization.behavior.existing-referent-anchoring.v1" +REFERENT_ANCHORING_CONFIG_KEY = "core.organization.existing_referent_anchoring" +REFERENT_ANCHORING_CONFIG_SCHEMA = "core.organization.existing_referent_anchoring.config.v1" + +DeploymentConfigManager.register_schema( + REFERENT_ANCHORING_CONFIG_SCHEMA, + BehaviorAgentConfig, + keys=(REFERENT_ANCHORING_CONFIG_KEY,), +) + + +class ExistingReferentAnchoringBehaviorResolver( + Resolver[str, str], + rso_type=REFERENT_ANCHORING_BEHAVIOR, +): + organization_description = ( + "Anchor one source-grounded referring fragment to existing " + "identity-bearing information." + ) + judgment_contract = ( + "The source expression meaningfully denotes a reusable referent.", + "Selected text identifies this mention without unrelated material.", + "The referent Block already exists and was not created as a temporary label.", + "The target contains enough identity to distinguish plausible alternatives.", + "Denotation remains continuous across name, time, environment, and scope.", + "Plausible competing referents have been considered and excluded.", + "The anchor improves cross-source or cross-time use rather than graph density.", + ) + + async def get_text( + self, + *, + context: typing.Literal["default", "lexical"] = "default", + refresh: bool = False, + materialize_missing: bool = True, + ) -> str: + del context, refresh, materialize_missing + return self.organization_description + + async def get_label(self, *, refresh: bool = False) -> str: + del refresh + return "organization behavior: existing referent anchoring" + + @classmethod + async def record_candidate( + cls, + block_id: BlockID, + ) -> CandidateWriteResult: + return await record_candidate(cls, block_id) + + @classmethod + async def anchor_existing_referent( + cls, + source_block_id: BlockID, + selected_text: str, + referent_block_id: BlockID, + ) -> ExistingReferentAnchorResult: + proposal = ExistingReferentAnchorProposal( + source_block_id=source_block_id, + selected_text=selected_text, + referent_block_id=referent_block_id, + ) + async with graph_uow() as uow: + await require_distinct_blocks( + proposal.source_block_id, proposal.referent_block_id, uow + ) + source = await uow.blocks.get(proposal.source_block_id) + if source is None: # pragma: no cover - require_distinct_blocks invariant + raise ValueError("Source Block does not exist") + + existing = await cls._existing_path( + proposal.source_block_id, + proposal.selected_text, + proposal.referent_block_id, + uow, + ) + if existing is not None: + fragment_id, has_mention, refers_to = existing + return ExistingReferentAnchorResult( + fragment_block_id=fragment_id, + fragment_created=False, + has_mention=( + relation_result(has_mention, False) if has_mention is not None else None + ), + refers_to=relation_result(refers_to, False), + ) + + source_is_fragment = ( + source.resolver == "core.text.v1" and source.content == proposal.selected_text + ) + if source_is_fragment: + fragment_id = proposal.source_block_id + fragment_created = False + has_mention_result = None + else: + fragment = await uow.blocks.create( + BlockForm(resolver="core.text.v1", content=proposal.selected_text) + ) + if fragment.id is None: # pragma: no cover - persisted Block invariant + raise RuntimeError("Persisted referring fragment has no ID") + fragment_id = fragment.id + fragment_created = True + has_mention, created = await fetchsert_relation( + proposal.source_block_id, + fragment_id, + HAS_MENTION_RELATION, + uow, + ) + has_mention_result = relation_result(has_mention, created) + + refers_to, created = await fetchsert_relation( + fragment_id, + proposal.referent_block_id, + REFERS_TO_RELATION, + uow, + ) + return ExistingReferentAnchorResult( + fragment_block_id=fragment_id, + fragment_created=fragment_created, + has_mention=has_mention_result, + refers_to=relation_result(refers_to, created), + ) + + @staticmethod + async def _existing_path( + source_block_id: BlockID, + selected_text: str, + referent_block_id: BlockID, + uow: GraphUnitOfWork, + ) -> tuple[BlockID, RelationModel | None, RelationModel] | None: + source = await uow.blocks.get(source_block_id) + if ( + source is not None + and source.resolver == "core.text.v1" + and source.content == selected_text + ): + refers_to = next( + ( + relation + for relation in ( + await uow.relations.get( + source_block_id, include_in=False, content=REFERS_TO_RELATION + ) + ) + if relation.to_ == referent_block_id + ), + None, + ) + if refers_to is not None: + return source_block_id, None, refers_to + + for has_mention in await uow.relations.get( + source_block_id, include_in=False, content=HAS_MENTION_RELATION + ): + fragment = await uow.blocks.get(has_mention.to_) + if ( + fragment is None + or fragment.resolver != "core.text.v1" + or fragment.content != selected_text + ): + continue + refers_to = next( + ( + relation + for relation in ( + await uow.relations.get( + has_mention.to_, include_in=False, content=REFERS_TO_RELATION + ) + ) + if relation.to_ == referent_block_id + ), + None, + ) + if refers_to is not None: + return has_mention.to_, has_mention, refers_to + return None + + @classmethod + async def can_run_automatic(cls) -> bool: + return await configured_agent_available( + REFERENT_ANCHORING_CONFIG_KEY, + BehaviorAgentConfig, + ) + + @classmethod + async def run_automatic(cls, max_seeds: int) -> None: + candidates = await candidate_seed_ids(cls, max_seeds) + recent = await recent_block_ids(max_seeds) + random = await random_block_ids(max_seeds) + seeds = merge_seed_categories(max_seeds, candidates, recent, random) + LOGGER.info( + "organization.seeds.selected", + extra={ + "behavior": cls.__rsotype__, + "max_seeds": max_seeds, + "candidate_count": len(candidates), + "recent_count": len(recent), + "random_count": len(random), + "seed_block_ids": seeds, + }, + ) + for seed in seeds: + with continue_after_seed_failure(LOGGER, cls.__rsotype__, seed): + message = await build_seed_message( + "Anchor only resolved source mentions to existing identity-bearing Blocks.", + cls.judgment_contract, + seed, + ) + if message is None: + LOGGER.info( + "organization.seed.considered", + extra={ + "behavior": cls.__rsotype__, + "seed_block_ids": (seed,), + "outcome": "unresolved", + "reason": "text_unavailable", + }, + ) + continue + await run_configured_agent( + REFERENT_ANCHORING_CONFIG_KEY, + BehaviorAgentConfig, + message, + ) + LOGGER.info( + "organization.seed.considered", + extra={ + "behavior": cls.__rsotype__, + "seed_block_ids": (seed,), + "outcome": "considered", + "reason": "agent_completed", + }, + ) diff --git a/inkcre_core/business/organization/refinement.py b/inkcre_core/business/organization/refinement.py new file mode 100644 index 00000000..5f44c26b --- /dev/null +++ b/inkcre_core/business/organization/refinement.py @@ -0,0 +1,187 @@ +"""Non-dominating refinement relation and automatic semantic judgment.""" + +from __future__ import annotations + +import typing + + +from inkcre_core.business.deployment_config import DeploymentConfigManager +from inkcre_core.business.info_base.resolver import Resolver +from inkcre_core.persistence.info_base.uow import GraphUnitOfWork, graph_uow +from inkcre_core.libs.obsrv.main import get_logger +from inkcre_core.schemas.info_base.block import BlockID +from inkcre_core.schemas.organization_behavior import ( + BehaviorAgentConfig, + CandidateWriteResult, + RelationWriteResult, +) + +from ._shared import ( + build_seed_message, + candidate_seed_ids, + configured_agent_available, + continue_after_seed_failure, + fetchsert_relation, + merge_seed_categories, + random_block_ids, + recent_block_ids, + recent_relation_endpoint_ids, + record_candidate, + relation_result, + require_distinct_blocks, + run_configured_agent, +) +from .supersession import EDITED_RELATION + + +LOGGER = get_logger().getChild(__name__) + +REFINES_RELATION = "refines" +REFINEMENT_BEHAVIOR = "core.organization.behavior.refinement.v1" +REFINEMENT_CONFIG_KEY = "core.organization.refinement" +REFINEMENT_CONFIG_SCHEMA = "core.organization.refinement.config.v1" + +DeploymentConfigManager.register_schema( + REFINEMENT_CONFIG_SCHEMA, BehaviorAgentConfig, keys=(REFINEMENT_CONFIG_KEY,) +) + + +class RefinementBehaviorResolver( + Resolver[str, str], + rso_type=REFINEMENT_BEHAVIOR, +): + organization_description = ( + "Relate useful compatible detail that refines but does not replace information." + ) + judgment_contract = ( + "Both endpoints are complete addressable information units.", + "They continue the same referent and evolvable subject.", + "The refinement scope equals or is visibly contained by the predecessor scope.", + "Their information roles and attribution remain compatible.", + "The refinement adds reusable detail, constraints, explanation, or precision.", + "The predecessor remains independently safe as a coarser description.", + ) + + async def get_text( + self, + *, + context: typing.Literal["default", "lexical"] = "default", + refresh: bool = False, + materialize_missing: bool = True, + ) -> str: + del context, refresh, materialize_missing + return self.organization_description + + async def get_label(self, *, refresh: bool = False) -> str: + del refresh + return "organization behavior: refinement" + + @classmethod + async def record_candidate( + cls, + block_id: BlockID, + ) -> CandidateWriteResult: + return await record_candidate(cls, block_id) + + @classmethod + async def record_refinement( + cls, + refinement_block_id: BlockID, + predecessor_block_id: BlockID, + ) -> RelationWriteResult: + async with graph_uow() as uow: + await require_distinct_blocks(refinement_block_id, predecessor_block_id, uow) + if await cls._has_directed_path( + predecessor_block_id, + refinement_block_id, + uow=uow, + ): + raise ValueError( + f"Cannot refine: an existing refines path runs from predecessor " + f"{predecessor_block_id} to refinement {refinement_block_id}" + ) + relation, created = await fetchsert_relation( + refinement_block_id, + predecessor_block_id, + REFINES_RELATION, + uow, + ) + return relation_result(relation, created) + + @classmethod + async def _has_directed_path( + cls, + start: BlockID, + target: BlockID, + *, + uow: GraphUnitOfWork, + ) -> bool: + frontier = {start} + visited = {start} + while frontier: + rows = tuple( + relation.to_ + for relation in await uow.relations.get_outgoing_many( + frontier, content=REFINES_RELATION + ) + ) + next_frontier = set(rows) - visited + if target in next_frontier: + return True + visited.update(next_frontier) + frontier = next_frontier + return False + + @classmethod + async def can_run_automatic(cls) -> bool: + return await configured_agent_available(REFINEMENT_CONFIG_KEY, BehaviorAgentConfig) + + @classmethod + async def run_automatic(cls, max_seeds: int) -> None: + candidates = await candidate_seed_ids(cls, max_seeds) + strong = merge_seed_categories( + max_seeds, + await recent_relation_endpoint_ids(max_seeds, contents=(EDITED_RELATION,)), + await recent_block_ids(max_seeds), + ) + random = await random_block_ids(max_seeds) + seeds = merge_seed_categories(max_seeds, candidates, strong, random) + LOGGER.info( + "organization.seeds.selected", + extra={ + "behavior": cls.__rsotype__, + "max_seeds": max_seeds, + "candidate_count": len(candidates), + "strong_count": len(strong), + "random_count": len(random), + "seed_block_ids": seeds, + }, + ) + for seed in seeds: + with continue_after_seed_failure(LOGGER, cls.__rsotype__, seed): + message = await build_seed_message( + "Determine only useful non-dominating refinement relations.", + cls.judgment_contract, + seed, + ) + if message is None: + LOGGER.info( + "organization.seed.considered", + extra={ + "behavior": cls.__rsotype__, + "seed_block_ids": (seed,), + "outcome": "unresolved", + "reason": "text_unavailable", + }, + ) + continue + await run_configured_agent(REFINEMENT_CONFIG_KEY, BehaviorAgentConfig, message) + LOGGER.info( + "organization.seed.considered", + extra={ + "behavior": cls.__rsotype__, + "seed_block_ids": (seed,), + "outcome": "considered", + "reason": "agent_completed", + }, + ) diff --git a/inkcre_core/business/organization/rumination.py b/inkcre_core/business/organization/rumination.py new file mode 100644 index 00000000..132ddece --- /dev/null +++ b/inkcre_core/business/organization/rumination.py @@ -0,0 +1,256 @@ +"""Explicit and automatic rumination carried by an exact Behavior Resolver.""" + +from __future__ import annotations + +import json +import typing + +import pydantic + +from inkcre_core.business.deployment_config import DeploymentConfigManager +from inkcre_core.business.info_base.resolver import ( + Resolver, + ResolverManager, + UnknownResolverError, + UnsupportedResolverCapability, +) +from inkcre_core.business.peer import PeerManager +from inkcre_core.persistence.info_base.uow import graph_uow +from inkcre_core.libs.obsrv.main import get_logger +from inkcre_core.schemas.ai import JSONValue, TextContentPart, UserMessage +from inkcre_core.schemas.info_base.block import BlockID +from inkcre_core.schemas.info_base.relation import RelationModel +from inkcre_core.schemas.organization import RuminationConfig, RuminationRequest +from inkcre_core.schemas.organization_behavior import CandidateWriteResult +from inkcre_core.schemas.peer import PeerProtocolRequest, PeerProtocolResponse, PeerRef + +from ._shared import ( + candidate_seed_ids, + configured_agent_available, + continue_after_seed_failure, + merge_seed_categories, + random_block_ids, + recent_block_ids, + record_candidate, + run_configured_agent, +) +from .contracts import ( + OrganizationBlockNotFoundError, + OrganizationDelegationError, + OrganizationExecutionError, +) + + +LOGGER = get_logger().getChild(__name__) + +RUMINATION_CONFIG_KEY = "core.organization.rumination" +RUMINATION_CONFIG_SCHEMA = "core.organization.rumination.config.v1" +RUMINATION_CAPABILITY = "core.organization.rumination.v1" +RUMINATION_BEHAVIOR = "core.organization.behavior.rumination.v1" + +DeploymentConfigManager.register_schema( + RUMINATION_CONFIG_SCHEMA, RuminationConfig, keys=(RUMINATION_CONFIG_KEY,) +) + + +class RuminationBehaviorResolver( + Resolver[str, str], + rso_type=RUMINATION_BEHAVIOR, +): + organization_description = ( + "Open-ended reconsideration of one information Block that may add a useful graph." + ) + + async def get_text( + self, + *, + context: typing.Literal["default", "lexical"] = "default", + refresh: bool = False, + materialize_missing: bool = True, + ) -> str: + del context, refresh, materialize_missing + return self.organization_description + + async def get_label(self, *, refresh: bool = False) -> str: + del refresh + return "organization behavior: rumination" + + @classmethod + async def record_candidate( + cls, + block_id: BlockID, + ) -> CandidateWriteResult: + return await record_candidate(cls, block_id) + + @classmethod + async def can_run_automatic(cls) -> bool: + return await configured_agent_available(RUMINATION_CONFIG_KEY, RuminationConfig) + + @classmethod + async def run_automatic(cls, max_seeds: int) -> None: + candidates = await candidate_seed_ids(cls, max_seeds) + recent = await recent_block_ids(max_seeds) + random = await random_block_ids(max_seeds) + seeds = merge_seed_categories(max_seeds, candidates, recent, random) + LOGGER.info( + "organization.seeds.selected", + extra={ + "behavior": cls.__rsotype__, + "max_seeds": max_seeds, + "candidate_count": len(candidates), + "recent_count": len(recent), + "random_count": len(random), + "seed_block_ids": seeds, + }, + ) + for seed in seeds: + with continue_after_seed_failure(LOGGER, cls.__rsotype__, seed): + await cls.ruminate_local(seed) + LOGGER.info( + "organization.seed.considered", + extra={ + "behavior": cls.__rsotype__, + "seed_block_ids": (seed,), + "outcome": "considered", + "reason": "agent_completed", + }, + ) + + @classmethod + async def ruminate( + cls, + block_id: BlockID, + *, + route_to_peer: PeerRef | None = None, + ) -> None: + """Execute locally unless the caller explicitly selects another Peer.""" + request = RuminationRequest(block=block_id) + if route_to_peer is None or route_to_peer == PeerManager.get_current_peer_ref(): + await cls.ruminate_local(request.block) + return + + payload = PeerProtocolRequest( + body=typing.cast(JSONValue, request.model_dump(mode="json")) + ) + result = await PeerManager.delegate( + RUMINATION_CAPABILITY, + typing.cast(JSONValue, payload.model_dump(mode="json", exclude_unset=True)), + route_to_peer=route_to_peer, + ) + try: + response = PeerProtocolResponse.model_validate(result) + except pydantic.ValidationError as error: + raise OrganizationDelegationError( + "Rumination Peer returned an invalid response" + ) from error + if response.status != 204 or "body" in response.model_fields_set: + raise OrganizationDelegationError(f"Rumination Peer returned HTTP {response.status}") + + @classmethod + async def ruminate_local(cls, block_id: BlockID) -> None: + initial_message = await cls._build_initial_message(block_id) + if initial_message is None: + return + await run_configured_agent( + RUMINATION_CONFIG_KEY, + RuminationConfig, + initial_message, + ) + + @classmethod + async def _build_initial_message(cls, block_id: BlockID) -> UserMessage | None: + async with graph_uow() as uow: + block = await uow.blocks.get(block_id) + if block is None: + raise OrganizationBlockNotFoundError(f"Block {block_id} does not exist") + relations = tuple( + sorted( + (await uow.relations.get(block_id)), + key=lambda relation: relation.id or 0, + ) + ) + neighbor_ids = { + relation.to_ if relation.from_ == block_id else relation.from_ + for relation in relations + } + neighbors = { + neighbor.id: neighbor for neighbor in await uow.blocks.get_many(neighbor_ids) + } + + try: + focal_text = await ResolverManager.get(block).get_text() + except (UnknownResolverError, UnsupportedResolverCapability): + return None + except Exception as error: + raise OrganizationExecutionError( + "Rumination could not understand focal Block" + ) from error + if focal_text is None or not focal_text.strip(): + return None + + relation_context: list[dict[str, typing.Any]] = [] + for relation in relations: + neighbor_id, direction = cls._neighbor_and_direction(block_id, relation) + neighbor = neighbors.get(neighbor_id) + label: str | None = None + if neighbor is not None: + try: + label = await ResolverManager.get(neighbor).get_label() + except Exception: + LOGGER.debug( + "Could not project rumination neighbor label", + exc_info=True, + extra={"block": neighbor_id}, + ) + relation_context.append( + { + "id": relation.id, + "direction": direction, + "property": relation.content, + "other_block": { + "id": neighbor_id, + "resolver": neighbor.resolver if neighbor is not None else None, + "label": label, + }, + } + ) + + context = { + "request": "ruminate", + "focal_block": { + "id": block_id, + "resolver": block.resolver, + "text": focal_text, + }, + "direct_relations": relation_context, + "available_draft_resolvers": [ + { + "resolver": capability.resolver, + "description": capability.description, + } + for capability in ResolverManager.get_draft_capabilities() + ], + } + return UserMessage( + content=( + TextContentPart( + text=json.dumps( + context, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ) + ), + ) + ) + + @staticmethod + def _neighbor_and_direction( + block_id: BlockID, + relation: RelationModel, + ) -> tuple[BlockID, typing.Literal["incoming", "outgoing", "self"]]: + if relation.from_ == block_id and relation.to_ == block_id: + return block_id, "self" + if relation.from_ == block_id: + return relation.to_, "outgoing" + return relation.from_, "incoming" diff --git a/inkcre_core/business/organization/supersession.py b/inkcre_core/business/organization/supersession.py new file mode 100644 index 00000000..753c7f3d --- /dev/null +++ b/inkcre_core/business/organization/supersession.py @@ -0,0 +1,305 @@ +"""Scoped supersession relation and automatic semantic judgment.""" + +from __future__ import annotations + +from collections import deque +from graphlib import CycleError, TopologicalSorter +import typing + + +from inkcre_core.business.deployment_config import DeploymentConfigManager +from inkcre_core.business.info_base.resolver import Resolver +from inkcre_core.persistence.info_base.uow import GraphUnitOfWork, graph_uow +from inkcre_core.libs.obsrv.main import get_logger +from inkcre_core.schemas.graph_navigation_retrieval import GraphModel +from inkcre_core.schemas.info_base.block import BlockID +from inkcre_core.schemas.info_base.relation import RelationID, RelationModel +from inkcre_core.schemas.organization_behavior import ( + BehaviorAgentConfig, + CandidateWriteResult, + RelationWriteResult, + SupersessionLineage, +) + +from ._shared import ( + build_seed_message, + candidate_seed_ids, + configured_agent_available, + continue_after_seed_failure, + fetchsert_relation, + merge_seed_categories, + random_block_ids, + recent_block_ids, + recent_relation_endpoint_ids, + record_candidate, + relation_result, + require_distinct_blocks, + run_configured_agent, +) + + +LOGGER = get_logger().getChild(__name__) + +SUPERSEDES_RELATION = "supersedes" +EDITED_RELATION = "edited" +SUPERSESSION_BEHAVIOR = "core.organization.behavior.supersession.v1" +SUPERSESSION_CONFIG_KEY = "core.organization.supersession" +SUPERSESSION_CONFIG_SCHEMA = "core.organization.supersession.config.v1" + +DeploymentConfigManager.register_schema( + SUPERSESSION_CONFIG_SCHEMA, + BehaviorAgentConfig, + keys=(SUPERSESSION_CONFIG_KEY,), +) + + +class SupersessionBehaviorResolver( + Resolver[str, str], + rso_type=SUPERSESSION_BEHAVIOR, +): + organization_description = ( + "Relate a semantic successor that fully replaces one predecessor in scope." + ) + judgment_contract = ( + "Both endpoints are complete addressable information units.", + "They continue the same referent and evolvable subject.", + "The successor covers the predecessor's complete applicable scope.", + "Semantic order, not collection time, identifies successor and predecessor.", + "The successor has authority for this subject and scope.", + "Continuing to use the predecessor as current would be wrong.", + ) + + async def get_text( + self, + *, + context: typing.Literal["default", "lexical"] = "default", + refresh: bool = False, + materialize_missing: bool = True, + ) -> str: + del context, refresh, materialize_missing + return self.organization_description + + async def get_label(self, *, refresh: bool = False) -> str: + del refresh + return "organization behavior: supersession" + + @classmethod + async def record_candidate( + cls, + block_id: BlockID, + ) -> CandidateWriteResult: + return await record_candidate(cls, block_id) + + @classmethod + async def record_supersession( + cls, + successor_block_id: BlockID, + predecessor_block_id: BlockID, + ) -> RelationWriteResult: + async with graph_uow() as uow: + await require_distinct_blocks(successor_block_id, predecessor_block_id, uow) + if await cls._has_directed_path( + predecessor_block_id, + successor_block_id, + uow=uow, + ): + raise ValueError( + f"Cannot supersede: an existing supersedes path runs from predecessor " + f"{predecessor_block_id} to successor {successor_block_id}" + ) + relation, created = await fetchsert_relation( + successor_block_id, + predecessor_block_id, + SUPERSEDES_RELATION, + uow, + ) + return relation_result(relation, created) + + @classmethod + async def _has_directed_path( + cls, + start: BlockID, + target: BlockID, + *, + uow: GraphUnitOfWork, + ) -> bool: + frontier = {start} + visited = {start} + while frontier: + rows = tuple( + relation.to_ + for relation in await uow.relations.get_outgoing_many( + frontier, content=SUPERSEDES_RELATION + ) + ) + next_frontier = set(rows) - visited + if target in next_frontier: + return True + visited.update(next_frontier) + frontier = next_frontier + return False + + async def read_lineage( + self, + focal_block_id: BlockID, + *, + max_explored_blocks: int = 1000, + max_explored_relations: int = 10000, + ) -> SupersessionLineage: + """Read supersedes history; incomplete or cyclic graphs have no current frontier.""" + if max_explored_blocks < 1 or max_explored_relations < 1: + raise ValueError("exploration bounds must be positive") + return await self._read_lineage( + focal_block_id, + max_explored_blocks=max_explored_blocks, + max_explored_relations=max_explored_relations, + ) + + async def _read_lineage( + self, + focal_block_id: BlockID, + *, + max_explored_blocks: int, + max_explored_relations: int, + ) -> SupersessionLineage: + async with graph_uow() as uow: + if (await uow.blocks.get(focal_block_id)) is None: + raise ValueError("Focal Block does not exist") + visited = {focal_block_id} + frontier = deque((focal_block_id,)) + relations: dict[RelationID, RelationModel] = {} + scanned = 0 + truncated = False + while frontier and not truncated: + current = frontier.popleft() + for endpoint in ("from", "to"): + cursor: RelationID | None = None + while not truncated: + remaining = max_explored_relations - scanned + if remaining == 0: + truncated = True + break + requested = min(200, remaining + 1) + page = await uow.relations.get_endpoint_page( + (current,), + endpoint=typing.cast(typing.Literal["from", "to"], endpoint), + contents=(SUPERSEDES_RELATION,), + cursor=cursor, + limit=requested, + ) + if len(page) > remaining: + page = page[:remaining] + truncated = True + scanned += len(page) + for relation in page: + if relation.id is None: + continue + relations[relation.id] = relation + neighbor = relation.to_ if relation.from_ == current else relation.from_ + if neighbor in visited: + continue + if len(visited) >= max_explored_blocks: + truncated = True + break + visited.add(neighbor) + frontier.append(neighbor) + if truncated or len(page) < requested: + break + cursor = typing.cast(RelationID, page[-1].id) + + cycle_detected = self._cycle_detected(visited, tuple(relations.values())) + incoming = {relation.to_ for relation in relations.values()} + current_block_ids = ( + () + if truncated or cycle_detected + else tuple(sorted(block_id for block_id in visited if block_id not in incoming)) + ) + blocks = await uow.blocks.get_many(visited) + existing = {block.id for block in blocks} + closed_relations = tuple( + relation + for relation in relations.values() + if relation.from_ in existing and relation.to_ in existing + ) + return SupersessionLineage( + graph=GraphModel(blocks=blocks, relations=closed_relations), + current_block_ids=current_block_ids, + truncated=truncated, + cycle_detected=cycle_detected, + ) + + @staticmethod + def _cycle_detected( + blocks: typing.Collection[BlockID], + relations: typing.Collection[RelationModel], + ) -> bool: + # The standard library uses an explicit stack, so long lineages do not + # consume Python call-stack depth. + sorter: TopologicalSorter[BlockID] = TopologicalSorter() + for block_id in blocks: + sorter.add(block_id) + for relation in relations: + sorter.add(relation.to_, relation.from_) + try: + sorter.prepare() + except CycleError: + return True + return False + + @classmethod + async def can_run_automatic(cls) -> bool: + return await configured_agent_available(SUPERSESSION_CONFIG_KEY, BehaviorAgentConfig) + + @classmethod + async def run_automatic(cls, max_seeds: int) -> None: + candidates = await candidate_seed_ids(cls, max_seeds) + strong = merge_seed_categories( + max_seeds, + await recent_relation_endpoint_ids(max_seeds, contents=(EDITED_RELATION,)), + await recent_block_ids(max_seeds), + ) + random = await random_block_ids(max_seeds) + seeds = merge_seed_categories(max_seeds, candidates, strong, random) + LOGGER.info( + "organization.seeds.selected", + extra={ + "behavior": cls.__rsotype__, + "max_seeds": max_seeds, + "candidate_count": len(candidates), + "strong_count": len(strong), + "random_count": len(random), + "seed_block_ids": seeds, + }, + ) + for seed in seeds: + with continue_after_seed_failure(LOGGER, cls.__rsotype__, seed): + message = await build_seed_message( + "Determine only well-supported scoped supersession relations.", + cls.judgment_contract, + seed, + ) + if message is None: + LOGGER.info( + "organization.seed.considered", + extra={ + "behavior": cls.__rsotype__, + "seed_block_ids": (seed,), + "outcome": "unresolved", + "reason": "text_unavailable", + }, + ) + continue + await run_configured_agent( + SUPERSESSION_CONFIG_KEY, + BehaviorAgentConfig, + message, + ) + LOGGER.info( + "organization.seed.considered", + extra={ + "behavior": cls.__rsotype__, + "seed_block_ids": (seed,), + "outcome": "considered", + "reason": "agent_completed", + }, + ) diff --git a/inkcre_core/business/organization/synthesis.py b/inkcre_core/business/organization/synthesis.py new file mode 100644 index 00000000..1a2523f7 --- /dev/null +++ b/inkcre_core/business/organization/synthesis.py @@ -0,0 +1,236 @@ +"""Provenance-preserving n-ary synthesis behavior.""" + +from __future__ import annotations + +import typing + + +from inkcre_core.business.deployment_config import DeploymentConfigManager +from inkcre_core.business.info_base.resolver import Resolver +from inkcre_core.persistence.info_base.uow import GraphUnitOfWork, graph_uow +from inkcre_core.libs.obsrv.main import get_logger +from inkcre_core.schemas.info_base.block import BlockForm, BlockID, BlockModel +from inkcre_core.schemas.organization_behavior import ( + BehaviorAgentConfig, + CandidateWriteResult, + SynthesisProposal, + SynthesisWriteResult, +) + +from ._shared import ( + build_seed_message, + candidate_seed_ids, + configured_agent_available, + continue_after_seed_failure, + fetchsert_relation, + merge_seed_categories, + random_block_ids, + recent_block_ids, + recent_relation_endpoint_ids, + record_candidate, + relation_result, + run_configured_agent, +) +from .supersession import EDITED_RELATION + + +LOGGER = get_logger().getChild(__name__) + +SYNTHESIS_RELATION = "synthesis" +SYNTHESIS_BEHAVIOR = "core.organization.behavior.synthesis.v1" +SYNTHESIS_CONFIG_KEY = "core.organization.synthesis" +SYNTHESIS_CONFIG_SCHEMA = "core.organization.synthesis.config.v1" + +DeploymentConfigManager.register_schema( + SYNTHESIS_CONFIG_SCHEMA, BehaviorAgentConfig, keys=(SYNTHESIS_CONFIG_KEY,) +) + + +class SynthesisBehaviorResolver( + Resolver[str, str], + rso_type=SYNTHESIS_BEHAVIOR, +): + organization_description = ( + "Create reusable multi-source information while preserving exact source basis." + ) + judgment_contract = ( + "The result is independently useful information, not a list of related sources.", + "Every source materially contributes content, attribution, or uncertainty.", + "Source disagreements, uncertainty, and speaker attribution remain visible.", + "Scopes are compatible or their differences are explicitly preserved.", + "Duplicate-connected copies do not multiply independent corroboration.", + "No existing synthesis already provides the same reusable distinction.", + "Observed past use and recurrence make future reuse plausible.", + ) + + async def get_text( + self, + *, + context: typing.Literal["default", "lexical"] = "default", + refresh: bool = False, + materialize_missing: bool = True, + ) -> str: + del context, refresh, materialize_missing + return self.organization_description + + async def get_label(self, *, refresh: bool = False) -> str: + del refresh + return "organization behavior: synthesis" + + @classmethod + async def record_candidate( + cls, + block_id: BlockID, + ) -> CandidateWriteResult: + return await record_candidate(cls, block_id) + + @classmethod + async def create_synthesis( + cls, + text: str, + source_block_ids: typing.Collection[BlockID], + previous_synthesis_block_id: BlockID | None = None, + ) -> SynthesisWriteResult: + proposal = SynthesisProposal( + text=text, + source_block_ids=tuple(source_block_ids), + previous_synthesis_block_id=previous_synthesis_block_id, + ) + async with graph_uow() as uow: + found = {block.id for block in (await uow.blocks.get_many(proposal.source_block_ids))} + missing = tuple(source for source in proposal.source_block_ids if source not in found) + if missing: + raise ValueError(f"Synthesis source Blocks do not exist: {missing!r}") + if proposal.previous_synthesis_block_id is not None: + previous = await uow.blocks.get(proposal.previous_synthesis_block_id) + if previous is None: + raise ValueError("Previous synthesis Block does not exist") + previous_basis = await cls._source_basis(previous, uow) + if len(set(previous_basis)) < 2: + raise ValueError("Previous synthesis has no valid multi-source basis") + + source_set = set(proposal.source_block_ids) + text_candidates = await uow.blocks.matching_content("core.text.v1", proposal.text) + synthesis = None + for candidate in text_candidates: + if await cls._source_basis(candidate, uow) == source_set: + synthesis = candidate + break + synthesis_created = synthesis is None + if synthesis is None: + synthesis = await uow.blocks.create( + BlockForm(resolver="core.text.v1", content=proposal.text) + ) + if synthesis.id is None: # pragma: no cover - persisted Block invariant + raise RuntimeError("Persisted synthesis Block has no ID") + + basis = [] + for source_block_id in sorted(source_set): + relation, created = await fetchsert_relation( + source_block_id, + synthesis.id, + SYNTHESIS_RELATION, + uow, + ) + basis.append(relation_result(relation, created)) + + edited = None + if ( + proposal.previous_synthesis_block_id is not None + and proposal.previous_synthesis_block_id != synthesis.id + ): + relation, created = await fetchsert_relation( + proposal.previous_synthesis_block_id, + synthesis.id, + EDITED_RELATION, + uow, + ) + edited = relation_result(relation, created) + return SynthesisWriteResult( + synthesis_block_id=synthesis.id, + synthesis_created=synthesis_created, + basis=tuple(basis), + edited=edited, + ) + + @staticmethod + async def _source_basis( + synthesis: BlockModel, + uow: GraphUnitOfWork, + ) -> set[BlockID]: + if synthesis.id is None: + return set() + return { + relation.from_ + for relation in await uow.relations.get( + synthesis.id, include_out=False, content=SYNTHESIS_RELATION + ) + } + + @classmethod + async def _change_signal_ids(cls, limit: int) -> tuple[BlockID, ...]: + endpoints = await recent_relation_endpoint_ids(limit) + if not endpoints: + return () + async with graph_uow() as uow: + affected = tuple( + relation.to_ + for relation in await uow.relations.get_outgoing_many( + endpoints, content=SYNTHESIS_RELATION + ) + ) + return tuple(dict.fromkeys((*endpoints, *affected))) + + @classmethod + async def can_run_automatic(cls) -> bool: + return await configured_agent_available(SYNTHESIS_CONFIG_KEY, BehaviorAgentConfig) + + @classmethod + async def run_automatic(cls, max_seeds: int) -> None: + candidates = await candidate_seed_ids(cls, max_seeds) + strong = merge_seed_categories( + max_seeds, + await cls._change_signal_ids(max_seeds), + await recent_block_ids(max_seeds), + ) + random = await random_block_ids(max_seeds) + seeds = merge_seed_categories(max_seeds, candidates, strong, random) + LOGGER.info( + "organization.seeds.selected", + extra={ + "behavior": cls.__rsotype__, + "max_seeds": max_seeds, + "candidate_count": len(candidates), + "strong_count": len(strong), + "random_count": len(random), + "seed_block_ids": seeds, + }, + ) + for seed in seeds: + with continue_after_seed_failure(LOGGER, cls.__rsotype__, seed): + message = await build_seed_message( + "Create only provenance-preserving, reusable multi-source synthesis.", + cls.judgment_contract, + seed, + ) + if message is None: + LOGGER.info( + "organization.seed.considered", + extra={ + "behavior": cls.__rsotype__, + "seed_block_ids": (seed,), + "outcome": "unresolved", + "reason": "text_unavailable", + }, + ) + continue + await run_configured_agent(SYNTHESIS_CONFIG_KEY, BehaviorAgentConfig, message) + LOGGER.info( + "organization.seed.considered", + extra={ + "behavior": cls.__rsotype__, + "seed_block_ids": (seed,), + "outcome": "considered", + "reason": "agent_completed", + }, + ) diff --git a/inkcre_core/business/organization/tools.py b/inkcre_core/business/organization/tools.py new file mode 100644 index 00000000..ad24740a --- /dev/null +++ b/inkcre_core/business/organization/tools.py @@ -0,0 +1,342 @@ +"""Owner-coherent Agent read tools and exact Organization mutation tools.""" + +from __future__ import annotations + +import typing + +import pydantic + +from inkcre_core.business.agent import AgentManager, ToolExecutionError +from inkcre_core.business.info_base import InfoBaseManager +from inkcre_core.business.info_base.commands import submit_graph as persist_submitted_graph +from inkcre_core.business.info_base.resolver import ( + ResolverDraftCapability, + ResolverManager, +) +from inkcre_core.schemas.ai import JSONValue +from inkcre_core.schemas.organization import ( + DraftGraphInput, + GetDraftGraphSchemaInput, + SubmitGraphInput, +) +from inkcre_core.schemas.organization_behavior import ( + DuplicateAssertionProposal, + EvidenceStanceProposal, + ExistingReferentAnchorProposal, + RecordOrganizationCandidateInput, + RefinementProposal, + SupersessionProposal, + SynthesisProposal, +) +from ._shared import behavior_resolver_classes, get_behavior_resolver +from .contracts import OrganizationError +from .duplicate_assertion import ( + DUPLICATES_ASSERTION_RELATION, + DuplicateAssertionBehaviorResolver, +) +from .evidence_stance import EvidenceStanceBehaviorResolver +from .referent_anchoring import ExistingReferentAnchoringBehaviorResolver +from .refinement import REFINES_RELATION, RefinementBehaviorResolver +from .supersession import SUPERSEDES_RELATION, SupersessionBehaviorResolver +from .synthesis import SynthesisBehaviorResolver + + +GET_DRAFT_GRAPH_SCHEMA_TOOL = "get_draft_graph_schema" +DRAFT_GRAPH_TOOL = "draft_graph" +SUBMIT_GRAPH_TOOL = "submit_graph" +RECORD_SUPERSESSION_TOOL = "record_supersession" +RECORD_REFINEMENT_TOOL = "record_refinement" +RECORD_EVIDENCE_STANCE_TOOL = "record_evidence_stance" +CREATE_SYNTHESIS_TOOL = "create_synthesis" +ANCHOR_EXISTING_REFERENT_TOOL = "anchor_existing_referent" +RECORD_DUPLICATE_ASSERTION_TOOL = "record_duplicate_assertion" +RECORD_ORGANIZATION_CANDIDATE_TOOL = "record_organization_candidate" + + +def _draft_capability_snapshot() -> dict[str, ResolverDraftCapability]: + return { + capability.resolver: capability + for capability in ResolverManager.get_draft_capabilities() + } + + +def _schema_discovery_input_model() -> type[pydantic.BaseModel]: + snapshot = _draft_capability_snapshot() + if not snapshot: # pragma: no cover - core.text.v1 is always registered + raise RuntimeError("No Resolver graph-drafting capability is registered") + + def add_exact_ids(schema: dict[str, typing.Any]) -> None: + schema["properties"]["resolver_types"]["items"] = { + "type": "string", + "enum": list(snapshot), + } + + class BoundGetDraftGraphSchemaInput(GetDraftGraphSchemaInput): + model_config = pydantic.ConfigDict( + extra="forbid", + frozen=True, + json_schema_extra=add_exact_ids, + ) + + @pydantic.field_validator("resolver_types") + @classmethod + def exact_resolvers(cls, resolvers: tuple[str, ...]) -> tuple[str, ...]: + unknown = tuple(resolver for resolver in resolvers if resolver not in snapshot) + if unknown: + raise ValueError(f"Unavailable draft Resolver IDs: {unknown!r}") + return resolvers + + return BoundGetDraftGraphSchemaInput + + +def _draft_graph_input_model() -> type[pydantic.BaseModel]: + snapshot = _draft_capability_snapshot() + if not snapshot: # pragma: no cover - core.text.v1 is always registered + raise RuntimeError("No Resolver graph-drafting capability is registered") + + def add_exact_ids(schema: dict[str, typing.Any]) -> None: + schema["properties"]["resolver_type"] = { + "type": "string", + "enum": list(snapshot), + } + + class BoundDraftGraphInput(DraftGraphInput): + model_config = pydantic.ConfigDict( + extra="forbid", + json_schema_extra=add_exact_ids, + ) + + @pydantic.field_validator("resolver_type") + @classmethod + def exact_resolver(cls, resolver: str) -> str: + if resolver not in snapshot: + raise ValueError(f"Unavailable draft Resolver ID: {resolver!r}") + return resolver + + @pydantic.model_validator(mode="after") + def validate_resolver_input(self) -> typing.Self: + capability = snapshot[self.resolver_type] + # Validate at the caller's actual path; an inner resolver_type error must + # not tell the Agent to remove its valid outer selector. + payload_model = pydantic.create_model( + "ResolverDraftPayload", input=(capability.input_model, ...) + ) + payload = payload_model.model_validate({"input": self.input}) + object.__setattr__(self, "_resolver_input", getattr(payload, "input")) + return self + + return BoundDraftGraphInput + + +@AgentManager.tool( + GET_DRAFT_GRAPH_SCHEMA_TOOL, + description="Describe graph-drafting inputs for selected Resolver types.", + input_model_factory=_schema_discovery_input_model, +) +async def get_draft_graph_schema(input: GetDraftGraphSchemaInput) -> JSONValue: + snapshot = _draft_capability_snapshot() + return { + "resolvers": [ + { + "resolver_type": resolver, + "description": snapshot[resolver].description, + "input_schema": typing.cast( + dict[str, JSONValue], + snapshot[resolver].input_model.model_json_schema(), + ), + } + for resolver in input.resolver_types + ] + } + + +@AgentManager.tool( + DRAFT_GRAPH_TOOL, + description="Draft one rooted GraphForm through an exact Resolver without persistence.", + input_model_factory=_draft_graph_input_model, +) +async def draft_graph(input: DraftGraphInput) -> JSONValue: + capability = ResolverManager.get_draft_capability(input.resolver_type) + resolver_input = typing.cast(pydantic.BaseModel, getattr(input, "_resolver_input")) + stars = capability.resolver_cls.create_graph(resolver_input) + graph = InfoBaseManager.normalize_graph(stars, input.local_block_id_start) + return typing.cast(JSONValue, graph.model_dump(mode="json")) + + +@AgentManager.tool( + SUBMIT_GRAPH_TOOL, + description="Persist one complete GraphForm and return local-to-persisted Block IDs.", +) +async def submit_graph(input: SubmitGraphInput) -> JSONValue: + result = await persist_submitted_graph(input.graph) + return typing.cast(JSONValue, result.model_dump(mode="json")) + + +def _candidate_input_model() -> type[pydantic.BaseModel]: + snapshot = {behavior.__rsotype__: behavior for behavior in behavior_resolver_classes()} + if not snapshot: + raise RuntimeError("No Organization Behavior Resolver is registered") + + def add_exact_behaviors(schema: dict[str, typing.Any]) -> None: + schema["properties"]["behavior"] = { + "oneOf": [ + { + "const": behavior_id, + "description": behavior.organization_description, + } + for behavior_id, behavior in snapshot.items() + ] + } + + class BoundRecordOrganizationCandidateInput(RecordOrganizationCandidateInput): + model_config = pydantic.ConfigDict( + extra="forbid", + frozen=True, + json_schema_extra=add_exact_behaviors, + ) + + @pydantic.field_validator("behavior") + @classmethod + def exact_behavior(cls, behavior: str) -> str: + if behavior not in snapshot: + raise ValueError( + f"Unavailable behavior {behavior!r}; available: {', '.join(snapshot)}" + ) + return behavior + + return BoundRecordOrganizationCandidateInput + + +async def _exact_result(operation: typing.Awaitable[pydantic.BaseModel]) -> JSONValue: + try: + result = await operation + except (ValueError, OrganizationError) as error: + raise ToolExecutionError( + {"error": type(error).__name__, "message": str(error)} + ) from error + return typing.cast(JSONValue, result.model_dump(mode="json")) + + +@AgentManager.tool( + RECORD_SUPERSESSION_TOOL, + description=( + f"Record {SUPERSEDES_RELATION!r}: the successor is authorized to replace " + "the predecessor across its entire scope on the same subject." + ), +) +async def record_supersession(input: SupersessionProposal) -> JSONValue: + return await _exact_result( + SupersessionBehaviorResolver.record_supersession( + input.successor_block_id, + input.predecessor_block_id, + ) + ) + + +@AgentManager.tool( + RECORD_REFINEMENT_TOOL, + description=( + f"Record {REFINES_RELATION!r}: new compatible detail at equal or narrower scope " + "on the same subject, not mere extraction or rewording. The predecessor " + "remains independently usable as a coarser description." + ), +) +async def record_refinement(input: RefinementProposal) -> JSONValue: + return await _exact_result( + RefinementBehaviorResolver.record_refinement( + input.refinement_block_id, + input.predecessor_block_id, + ) + ) + + +@AgentManager.tool( + RECORD_EVIDENCE_STANCE_TOOL, + description=( + "Record attributable evidence supporting or challenging a whole " + "assertion in comparable scope, without declaring it true or false. " + "Merely establishing that derived content faithfully restates its source is " + "not evidence stance, regardless of source authority. Shared provenance is " + "allowed when observation or reasoning " + "contributes reasons beyond restatement." + ), +) +async def record_evidence_stance(input: EvidenceStanceProposal) -> JSONValue: + return await _exact_result( + EvidenceStanceBehaviorResolver.record_evidence_stance( + input.evidence_block_id, + input.assertion_block_id, + input.stance, + ) + ) + + +@AgentManager.tool( + CREATE_SYNTHESIS_TOOL, + description=( + "Create reusable multi-source information preserving provenance, " + "disagreement, uncertainty and speaker attribution; copies do not " + "multiply corroboration." + ), +) +async def create_synthesis(input: SynthesisProposal) -> JSONValue: + return await _exact_result( + SynthesisBehaviorResolver.create_synthesis( + input.text, + input.source_block_ids, + input.previous_synthesis_block_id, + ) + ) + + +@AgentManager.tool( + ANCHOR_EXISTING_REFERENT_TOOL, + description=( + "Link a source's referring fragment to an existing identity-bearing " + "referent, without creating a new referent." + ), +) +async def anchor_existing_referent( + input: ExistingReferentAnchorProposal, +) -> JSONValue: + return await _exact_result( + ExistingReferentAnchoringBehaviorResolver.anchor_existing_referent( + input.source_block_id, + input.selected_text, + input.referent_block_id, + ) + ) + + +@AgentManager.tool( + RECORD_DUPLICATE_ASSERTION_TOOL, + description=( + f"Record {DUPLICATES_ASSERTION_RELATION!r}: whole assertions from the same " + "provenance occurrence with no " + "independent evidence, reasoning, decision or material gain; matching " + "words alone are insufficient." + ), +) +async def record_duplicate_assertion(input: DuplicateAssertionProposal) -> JSONValue: + return await _exact_result( + DuplicateAssertionBehaviorResolver.record_duplicate_assertion( + input.left_block_id, + input.right_block_id, + ) + ) + + +@AgentManager.tool( + RECORD_ORGANIZATION_CANDIDATE_TOOL, + description="Mark an organization candidate without executing the behavior.", + input_model_factory=_candidate_input_model, +) +async def record_organization_candidate( + input: RecordOrganizationCandidateInput, +) -> JSONValue: + behavior = get_behavior_resolver(input.behavior) + if behavior is None: + raise ToolExecutionError({"error": "behavior_unavailable"}) + return await _exact_result( + typing.cast(typing.Any, behavior).record_candidate(input.block_id) + ) diff --git a/inkcre_core/business/organization_job.py b/inkcre_core/business/organization_job.py new file mode 100644 index 00000000..404def26 --- /dev/null +++ b/inkcre_core/business/organization_job.py @@ -0,0 +1,33 @@ +"""Exact automatic Organization commands hosted by Jobs.""" + +from inkcre_core.business.job import JobHandler +from inkcre_core.business.organization_media import ( + MEDIA_INTERPRETATION_JOB_TYPE, + can_handle_media_interpretation, + interpret_missing_media, +) +from inkcre_core.schemas.job import JobModel +from inkcre_core.schemas.organization import MediaInterpretationJobParameters + + +class MediaInterpretationJobHandler( + JobHandler[MediaInterpretationJobParameters], + job_type=MEDIA_INTERPRETATION_JOB_TYPE, + description="Interpret missing image, audio, and video Blocks through configured Agents.", + parameters_model=MediaInterpretationJobParameters, + default_timeout_seconds=1800, +): + @classmethod + async def can_handle(cls, parameters: MediaInterpretationJobParameters) -> bool: + del parameters + return await can_handle_media_interpretation() + + @classmethod + async def handle( + cls, + job: JobModel, + parameters: MediaInterpretationJobParameters, + ) -> None: + del parameters + report = await interpret_missing_media() + job.state = report.model_dump(mode="json") diff --git a/inkcre_core/business/organization_media.py b/inkcre_core/business/organization_media.py new file mode 100644 index 00000000..5daac8f1 --- /dev/null +++ b/inkcre_core/business/organization_media.py @@ -0,0 +1,266 @@ +"""System-driven media interpretation approach owned by Organization.""" + +import json +import logging +import typing + + +from inkcre_core.business.agent import AgentManager, TurnTermination +from inkcre_core.business.deployment_config import ( + DeploymentConfigManager, + DeploymentConfigService, +) +from inkcre_core.business.info_base.commands import get_related_block +from inkcre_core.business.info_base.resolver import ResolverManager +from inkcre_core.business.info_base.resolver.audio import AudioSolvedContent +from inkcre_core.business.info_base.resolver.image import ImageSolvedContent +from inkcre_core.business.info_base.resolver.video import VideoSolvedContent +from inkcre_core.persistence.info_base.uow import graph_uow +from inkcre_core.schemas.ai import ( + AudioContentPart, + ImageContentPart, + TextContentPart, + UserContentPart, + UserMessage, + VideoContentPart, +) +from inkcre_core.schemas.info_base.block import BlockModel +from inkcre_core.schemas.organization import ( + MediaInterpretationConfig, + MediaInterpretationDiagnostic, + MediaInterpretationReport, +) + + +logger = logging.getLogger(__name__) + +MEDIA_INTERPRETATION_CONFIG_KEY = "core.organization.media_interpretation" +MEDIA_INTERPRETATION_CONFIG_SCHEMA = "core.organization.media_interpretation.config.v1" +MEDIA_INTERPRETATION_JOB_TYPE = "core.organization.media_interpretation.v1" +_CANDIDATE_LIMIT = 20 +_DIAGNOSTIC_LIMIT = 20 +_RESOLVER_MODALITIES: dict[str, typing.Literal["image", "audio", "video"]] = { + "core.image.v1": "image", + "core.audio.v1": "audio", + "core.video.v1": "video", +} + + +DeploymentConfigManager.register_schema( + MEDIA_INTERPRETATION_CONFIG_SCHEMA, + MediaInterpretationConfig, + keys=(MEDIA_INTERPRETATION_CONFIG_KEY,), +) + + +async def _config() -> MediaInterpretationConfig | None: + value = await DeploymentConfigService.get(MEDIA_INTERPRETATION_CONFIG_KEY) + if value is None: + return None + if not isinstance(value, MediaInterpretationConfig): + raise TypeError("Media interpretation config registry returned the wrong model") + return value + + +def _agent( + config: MediaInterpretationConfig, + modality: typing.Literal["image", "audio", "video"], +) -> int: + return { + "image": config.image_agent, + "audio": config.audio_agent, + "video": config.video_agent, + }[modality] + + +async def can_handle_media_interpretation() -> bool: + config = await _config() + if config is None: + return False + for modality in typing.cast( + tuple[typing.Literal["image", "audio", "video"], ...], + ("image", "audio", "video"), + ): + if await AgentManager.can_execute(_agent(config, modality), modality): + return True + return False + + +async def _candidates() -> tuple[BlockModel, ...]: + async with graph_uow() as uow: + return await uow.blocks.without_outgoing_relation( + tuple(_RESOLVER_MODALITIES), "interpretation", limit=_CANDIDATE_LIMIT + ) + + +async def _relation_context(block_id: int) -> list[dict[str, typing.Any]]: + async with graph_uow() as uow: + relations = (await uow.relations.get(block_id))[:20] + neighbor_ids = { + relation.to_ if relation.from_ == block_id else relation.from_ + for relation in relations + } + neighbors = {block.id: block for block in await uow.blocks.get_many(neighbor_ids)} + context: list[dict[str, typing.Any]] = [] + for relation in relations: + other_id = relation.to_ if relation.from_ == block_id else relation.from_ + other = neighbors.get(other_id) + label = None + if other is not None: + try: + label = await ResolverManager.get(other).get_label() + except Exception: + logger.debug("Could not project media neighbor label", exc_info=True) + context.append( + { + "direction": "outgoing" if relation.from_ == block_id else "incoming", + "property": relation.content, + "other_block": {"id": other_id, "label": label}, + } + ) + return context + + +async def _message(block: BlockModel) -> UserMessage | None: + resolver = ResolverManager.get(block) + solved = await resolver.get_solved_content(materialize_missing=False) + transfer_url = await resolver.get_transfer_url() + media: UserContentPart + facts: dict[str, typing.Any] + if isinstance(solved, ImageSolvedContent): + media_type = solved.detected_media_type or ( + f"image/{solved.format}" if solved.format is not None else None + ) + if media_type is None: + return None + media = ImageContentPart( + data=solved.content, + mime_type=media_type, + transfer_url=transfer_url, + ) + facts = {"format": solved.format, "width": solved.width, "height": solved.height} + elif isinstance(solved, AudioSolvedContent): + media_type = solved.detected_media_type or ( + f"audio/{solved.container}" if solved.container is not None else None + ) + if media_type is None: + return None + media = AudioContentPart( + data=solved.content, + mime_type=media_type, + transfer_url=transfer_url, + ) + facts = { + "container": solved.container, + "codec": solved.codec, + "duration_ms": solved.duration_ms, + } + elif isinstance(solved, VideoSolvedContent): + media_type = solved.detected_media_type or ( + f"video/{solved.container}" if solved.container is not None else None + ) + if media_type is None: + return None + media = VideoContentPart( + data=solved.content, + mime_type=media_type, + transfer_url=transfer_url, + ) + facts = { + "container": solved.container, + "codec": solved.video_codec, + "duration_ms": solved.duration_ms, + "width": solved.width, + "height": solved.height, + } + else: + return None + + context = { + "request": "interpret media and submit only a useful additive graph", + "focal_block": { + "id": block.id, + "resolver": block.resolver, + "facts": facts, + }, + "direct_relations": await _relation_context(typing.cast(int, block.id)), + } + return UserMessage( + content=( + TextContentPart( + text=json.dumps( + context, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ) + ), + media, + ) + ) + + +async def interpret_missing_media() -> MediaInterpretationReport: + config = await _config() + if config is None: + return MediaInterpretationReport() + selected = interpreted = unavailable = failed = no_output = 0 + diagnostics: list[MediaInterpretationDiagnostic] = [] + + def diagnostic( + block: int, + modality: typing.Literal["image", "audio", "video"], + outcome: typing.Literal["unavailable", "failed", "no_output"], + reason: str, + ) -> None: + if len(diagnostics) < _DIAGNOSTIC_LIMIT: + diagnostics.append( + MediaInterpretationDiagnostic( + block=block, + modality=modality, + outcome=outcome, + reason=reason, + ) + ) + + for block in await _candidates(): + selected += 1 + block_id = typing.cast(int, block.id) + modality = _RESOLVER_MODALITIES[block.resolver] + agent = _agent(config, modality) + if not await AgentManager.can_execute(agent, modality): + unavailable += 1 + diagnostic(block_id, modality, "unavailable", "agent_not_locally_executable") + continue + try: + message = await _message(block) + if message is None: + unavailable += 1 + diagnostic(block_id, modality, "unavailable", "media_input_unavailable") + continue + thread = await AgentManager.run(agent, message) + turn = thread.current_turn + if turn is None: + raise RuntimeError("Media interpretation Agent did not start a Turn") + outcome = await turn + if outcome == TurnTermination.MAX_MODEL_CALLS: + raise RuntimeError("Media interpretation exceeded its model-call budget") + except Exception as error: + failed += 1 + logger.exception("Media interpretation failed", extra={"block": block_id}) + diagnostic(block_id, modality, "failed", type(error).__name__) + continue + if await get_related_block(block_id, content="interpretation") is None: + no_output += 1 + diagnostic(block_id, modality, "no_output", "missing_interpretation_relation") + else: + interpreted += 1 + + return MediaInterpretationReport( + selected=selected, + interpreted=interpreted, + unavailable=unavailable, + failed=failed, + no_output=no_output, + diagnostics=tuple(diagnostics), + ) diff --git a/app/business/peer/AGENTS.md b/inkcre_core/business/peer/AGENTS.md similarity index 100% rename from app/business/peer/AGENTS.md rename to inkcre_core/business/peer/AGENTS.md diff --git a/inkcre_core/business/peer/__init__.py b/inkcre_core/business/peer/__init__.py new file mode 100644 index 00000000..1be40f31 --- /dev/null +++ b/inkcre_core/business/peer/__init__.py @@ -0,0 +1,26 @@ +"""Peer discovery and request-response capability delegation.""" + +from .contracts import ( + CapabilityDelegationUnavailable, + DuplicatePeerRegistrationError, + PeerError, + PeerOutcomeUnknown, + PeerProtocolConfigurationError, + PeerProtocolError, + PeerRequestNotExecuted, +) +from .http import PeerHTTPInbound, PeerHTTPOutbound +from .main import PeerManager + +__all__ = [ + "CapabilityDelegationUnavailable", + "DuplicatePeerRegistrationError", + "PeerError", + "PeerHTTPInbound", + "PeerHTTPOutbound", + "PeerManager", + "PeerOutcomeUnknown", + "PeerProtocolConfigurationError", + "PeerProtocolError", + "PeerRequestNotExecuted", +] diff --git a/inkcre_core/business/peer/contracts.py b/inkcre_core/business/peer/contracts.py new file mode 100644 index 00000000..1f2df364 --- /dev/null +++ b/inkcre_core/business/peer/contracts.py @@ -0,0 +1,68 @@ +"""Peer discovery, routing, and outbound failure contracts.""" + +from collections.abc import Callable +from dataclasses import dataclass +import typing + +from inkcre_core.schemas.ai import JSONValue +from inkcre_core.schemas.peer import ( + CapabilityID, + CorePeerConfig, + PeerCapabilityAdvertisement, + PeerModel, + PeerProtocolID, +) + + +class PeerError(RuntimeError): + """Base failure exposed by the Peer domain.""" + + +class DuplicatePeerRegistrationError(PeerError): + pass + + +class CapabilityDelegationUnavailable(PeerError): + pass + + +class PeerProtocolError(PeerError): + pass + + +class PeerProtocolConfigurationError(PeerProtocolError): + """One advertised inbound cannot construct its caller-local outbound.""" + + +class PeerRequestNotExecuted(PeerProtocolError): + """The selected outbound proved capability execution did not begin.""" + + +class PeerOutcomeUnknown(PeerProtocolError): + """Dispatch may have occurred but no execution outcome is known.""" + + +class PeerInbound(typing.Protocol): + @property + def capability(self) -> CapabilityID: ... + + def advertise( + self, + config: CorePeerConfig, + ) -> PeerCapabilityAdvertisement | None: ... + + +class PeerOutbound(typing.Protocol): + async def execute(self, payload: JSONValue) -> JSONValue: ... + + +PeerOutboundFactory: typing.TypeAlias = Callable[ + [PeerModel, dict[str, JSONValue]], + PeerOutbound, +] + + +@dataclass(frozen=True) +class PeerOutboundRegistration: + protocol: PeerProtocolID + factory: PeerOutboundFactory diff --git a/inkcre_core/business/peer/http.py b/inkcre_core/business/peer/http.py new file mode 100644 index 00000000..e1dd8684 --- /dev/null +++ b/inkcre_core/business/peer/http.py @@ -0,0 +1,201 @@ +"""core.peer.protocol.http.v1 inbound advertisement and one-shot outbound.""" + +from __future__ import annotations + +from collections import defaultdict +from dataclasses import dataclass +import json +import typing + +import httpx +import pydantic + +from inkcre_core.auth import create_peer_jwt +from inkcre_core.schemas.ai import JSONValue +from inkcre_core.schemas.peer import ( + PEER_EXECUTION_HEADER, + PEER_HTTP_PROTOCOL, + PEER_NOT_EXECUTED, + CapabilityID, + CorePeerConfig, + PeerCapabilityAdvertisement, + PeerHTTPInboundParameters, + PeerInboundInterface, + PeerModel, + PeerProtocolRequest, + PeerProtocolResponse, +) +from inkcre_core.context import current_core +from inkcre_core.libs.obsrv.propagation import inject_context +from inkcre_core.libs.obsrv.telemetry import operation +from opentelemetry.trace import SpanKind + +from .contracts import ( + PeerOutcomeUnknown, + PeerProtocolConfigurationError, + PeerProtocolError, + PeerRequestNotExecuted, +) + + +_RESERVED_REQUEST_HEADERS = frozenset( + { + "authorization", + "connection", + "content-length", + "host", + "keep-alive", + "proxy-authenticate", + "proxy-authorization", + "te", + "trailer", + "transfer-encoding", + "upgrade", + } +) + + +@dataclass(frozen=True) +class PeerHTTPInbound: + """One fixed Business route advertised through Peer HTTP v1.""" + + capability: CapabilityID + method: str + path: str + + def __post_init__(self) -> None: + if not self.path.startswith("/") or "?" in self.path or "#" in self.path: + raise ValueError("Peer HTTP inbound path must be one absolute path") + + def advertise( + self, + config: CorePeerConfig, + ) -> PeerCapabilityAdvertisement | None: + if config.http_public_base_url is None: + return None + parameters = PeerHTTPInboundParameters( + method=self.method, + url=f"{config.http_public_base_url}{self.path}", + ) + return PeerCapabilityAdvertisement( + id=self.capability, + inbound=PeerInboundInterface( + protocol=PEER_HTTP_PROTOCOL, + parameters=typing.cast( + dict[str, JSONValue], + parameters.model_dump(mode="json"), + ), + ), + ) + + +class PeerHTTPOutbound: + """One-shot authenticated JSON HTTP execution for one advertised inbound.""" + + def __init__( + self, + peer: PeerModel, + parameters: dict[str, JSONValue], + ) -> None: + self.peer = peer + try: + self.parameters = PeerHTTPInboundParameters.model_validate(parameters) + except pydantic.ValidationError as error: + raise PeerProtocolConfigurationError( + f"Peer {peer.id} published invalid HTTP inbound parameters" + ) from error + + async def execute(self, payload: JSONValue) -> JSONValue: + try: + request = PeerProtocolRequest.model_validate(payload) + except pydantic.ValidationError as error: + raise PeerProtocolError("Invalid Peer HTTP request envelope") from error + + reserved = _RESERVED_REQUEST_HEADERS.intersection(request.headers) + if reserved: + rendered = ", ".join(sorted(reserved)) + raise PeerProtocolError(f"Peer HTTP payload attempted reserved headers: {rendered}") + + query = [(name, value) for name, values in request.query.items() for value in values] + headers = [ + (name, value) for name, values in request.headers.items() for value in values + ] + options = current_core().options + token = create_peer_jwt( + options.jwt_secret, lifetime_seconds=options.jwt_default_lifetime_seconds + ) + headers.append(("authorization", f"Bearer {token}")) + kwargs: dict[str, typing.Any] = { + "method": self.parameters.method, + "url": self.parameters.url, + "params": query, + "headers": headers, + } + if "body" in request.model_fields_set: + kwargs["json"] = request.body + + with operation( + "peer.http", + attributes={"inkcre.peer.target_id": str(self.peer.id)}, + kind=SpanKind.CLIENT, + ) as observation: + span = observation.span + propagated = inject_context() + if propagated: + # The transport owns Trace Context; business payload headers cannot duplicate it. + headers[:] = [ + (key, value) + for key, value in headers + if key.lower() not in {"traceparent", "tracestate"} + ] + headers.extend(propagated.items()) + try: + async with httpx.AsyncClient( + timeout=current_core().options.peer_http_timeout_seconds, + ) as client: + response = await client.request(**kwargs) + except (httpx.ConnectError, httpx.ConnectTimeout, httpx.PoolTimeout) as error: + span.set_attribute("inkcre.peer.outcome", "not_executed") + raise PeerRequestNotExecuted( + f"Could not dispatch to Peer {self.peer.id}" + ) from error + except httpx.RequestError as error: + span.set_attribute("inkcre.peer.outcome", "unknown") + raise PeerOutcomeUnknown( + f"Peer {self.peer.id} dispatch outcome is unknown" + ) from error + + if ( + response.headers.get(PEER_EXECUTION_HEADER, "").strip().lower() == PEER_NOT_EXECUTED + ): + span.set_attribute("inkcre.peer.outcome", "not_executed") + raise PeerRequestNotExecuted( + f"Peer {self.peer.id} reported that execution did not begin" + ) + + span.set_attribute("inkcre.peer.outcome", "responded") + span.set_attribute("http.response.status_code", response.status_code) + if response.status_code >= 500: + observation.outcome = "error" + + grouped_headers: defaultdict[str, list[str]] = defaultdict(list) + for name, value in response.headers.multi_items(): + if name.lower() != PEER_EXECUTION_HEADER.lower(): + grouped_headers[name.lower()].append(value) + + response_fields: dict[str, typing.Any] = { + "status": response.status_code, + "headers": grouped_headers, + } + if response.content: + try: + response_fields["body"] = response.json() + except json.JSONDecodeError as error: + raise PeerProtocolError( + f"Peer {self.peer.id} returned a non-JSON HTTP body" + ) from error + normalized = PeerProtocolResponse.model_validate(response_fields) + return typing.cast( + JSONValue, + normalized.model_dump(mode="json", exclude_unset=True), + ) diff --git a/inkcre_core/business/peer/main.py b/inkcre_core/business/peer/main.py new file mode 100644 index 00000000..a3d65d30 --- /dev/null +++ b/inkcre_core/business/peer/main.py @@ -0,0 +1,252 @@ +"""Peer identity, local inbound publication, discovery, and delegation.""" + +from __future__ import annotations + +from dataclasses import dataclass +import datetime +import logging +import random +import typing + +import pydantic + +from inkcre_core.configuration import ConfigContract +from inkcre_core.schemas.ai import JSONValue +from inkcre_core.schemas.peer import ( + PEER_HTTP_PROTOCOL, + CapabilityID, + CorePeerConfig, + PeerCapabilityAdvertisement, + PeerModel, + PeerRef, + normalize_capability_snapshot, +) +from inkcre_core.context import current_core + +from .contracts import ( + CapabilityDelegationUnavailable, + DuplicatePeerRegistrationError, + PeerInbound, + PeerOutcomeUnknown, + PeerOutboundFactory, + PeerProtocolConfigurationError, + PeerRequestNotExecuted, +) +from .http import PeerHTTPOutbound +from inkcre_core.persistence.peer.uow import peer_uow + + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class _Candidate: + peer: PeerModel + advertisement: PeerCapabilityAdvertisement + + +class PeerManager: + """Own Peer facts, local inbound registry, discovery, and one-shot routing.""" + + _random: random.Random = random.SystemRandom() + _config_contract = ConfigContract(CorePeerConfig) + + @classmethod + async def register_self(cls) -> PeerModel: + """Upsert runtime-owned identity/schema without changing owner configuration.""" + async with peer_uow() as peers: + peer = await peers.register( + current_core().options.peer_id, + current_core().options.peer_name, + current_core().options.application_version, + cls._config_contract.json_schema(), + ) + logger.info("Peer registered", extra={"peer": str(peer.id), "name": peer.name}) + return peer + + @classmethod + def get_current_peer_ref(cls) -> PeerRef: + return current_core().options.peer_id + + @classmethod + async def get_current_config_async(cls) -> CorePeerConfig: + peer = await cls.get_async(cls.get_current_peer_ref()) + if peer is None: + raise RuntimeError("Current Peer must be registered before reading config") + try: + return cls._config_contract.validate(peer.config) + except pydantic.ValidationError as error: + raise ValueError("Current Peer config is invalid") from error + + @classmethod + async def get_async(cls, peer: PeerRef) -> PeerModel | None: + async with peer_uow() as peers: + return await peers.get(peer) + + @classmethod + async def get_all(cls) -> tuple[PeerModel, ...]: + async with peer_uow() as peers: + return await peers.get_all() + + @classmethod + async def get_with_lease(cls, peer: PeerRef) -> dict[str, typing.Any] | None: + async with peer_uow() as peers: + rows, _ = await peers.with_leases(peer_id=peer) + return rows[0] if rows else None + + @classmethod + async def list_with_leases( + cls, *, limit: int | None = None, cursor: PeerRef | None = None + ) -> tuple[list[dict[str, typing.Any]], PeerRef | None]: + async with peer_uow() as peers: + return await peers.with_leases(limit=limit, cursor=cursor) + + @classmethod + def register_inbound(cls, inbound: PeerInbound) -> bool: + """Register one inbound and report whether this call added it.""" + existing = current_core().inbounds.get(inbound.capability) + if existing is inbound or existing == inbound: + return False + if existing is not None: + raise DuplicatePeerRegistrationError( + f"Capability inbound {inbound.capability!r} is already registered" + ) + current_core().inbounds[inbound.capability] = inbound + return True + + @classmethod + def unregister_inbound(cls, capability: CapabilityID) -> None: + current_core().inbounds.pop(capability, None) + + @classmethod + def register_outbound( + cls, + protocol: str, + factory: PeerOutboundFactory, + ) -> None: + existing = current_core().outbounds.get(protocol) + if existing is factory: + return + if existing is not None: + raise DuplicatePeerRegistrationError( + f"Peer outbound {protocol!r} is already registered" + ) + current_core().outbounds[protocol] = factory + + @classmethod + def setup_builtin_outbounds(cls) -> None: + cls.register_outbound(PEER_HTTP_PROTOCOL, PeerHTTPOutbound) + + @classmethod + async def publish_self(cls) -> PeerModel: + """Replace this Peer-owned capability snapshot from current config/registry.""" + async with peer_uow() as peers: + peer = await peers.get(current_core().options.peer_id, for_update=True) + if peer is None: + raise RuntimeError("Current Peer must be registered before publication") + try: + config = cls._config_contract.validate(peer.config) + except pydantic.ValidationError as error: + raise ValueError("Current Peer config is invalid") from error + advertisements = normalize_capability_snapshot( + advertisement + for inbound in current_core().inbounds.values() + if (advertisement := inbound.advertise(config)) is not None + ) + normalized = [ + typing.cast(dict[str, JSONValue], advertisement.model_dump(mode="json")) + for advertisement in advertisements + ] + if peer.capabilities != normalized: + peer.capabilities = normalized + await peers.save(peer) + return peer + + @classmethod + async def renew_self_lease(cls, ttl_seconds: int) -> datetime.datetime: + """Renew liveness through the database-time helper.""" + if ttl_seconds <= 0: + raise ValueError("Peer lease TTL must be positive") + async with peer_uow() as peers: + expiry = await peers.renew_lease(current_core().options.peer_id, ttl_seconds) + return expiry + + @classmethod + async def refresh_self(cls, ttl_seconds: int) -> PeerModel: + """Refresh advertisement, then renew its route lease.""" + peer = await cls.publish_self() + peer.lease_expires_at = await cls.renew_self_lease(ttl_seconds) + return peer + + @classmethod + async def clear_self_lease(cls) -> None: + async with peer_uow() as peers: + await peers.clear_lease(current_core().options.peer_id) + + @classmethod + async def delegate( + cls, + capability: CapabilityID, + payload: JSONValue, + *, + route_to_peer: PeerRef | None = None, + ) -> JSONValue: + """Execute through one eligible Peer, failing over only after non-execution.""" + candidates = await cls._candidates(capability, route_to_peer) + attempted = 0 + for candidate in candidates: + factory = current_core().outbounds.get(candidate.advertisement.inbound.protocol) + if factory is None: # candidate eligibility is rechecked defensively + continue + try: + outbound = factory( + candidate.peer, + candidate.advertisement.inbound.parameters, + ) + except PeerProtocolConfigurationError: + continue + attempted += 1 + try: + return await outbound.execute(payload) + except PeerRequestNotExecuted: + if route_to_peer is not None: + break + continue + except PeerOutcomeUnknown: + raise + raise CapabilityDelegationUnavailable( + f"No eligible Peer completed capability {capability!r}; attempted={attempted}" + ) + + @classmethod + async def _candidates( + cls, + capability: CapabilityID, + route_to_peer: PeerRef | None, + ) -> tuple[_Candidate, ...]: + async with peer_uow() as peers_repository: + peers = await peers_repository.candidates( + current_core().options.peer_id, route_to_peer + ) + + candidates: list[_Candidate] = [] + for peer in peers: + try: + snapshots = peer.capability_snapshot() + except (pydantic.ValidationError, ValueError): + logger.warning( + "Skipping malformed Peer capability snapshot", + extra={"peer": str(peer.id)}, + ) + continue + for advertisement in snapshots: + if ( + advertisement.id == capability + and advertisement.inbound.protocol in current_core().outbounds + ): + candidates.append(_Candidate(peer, advertisement)) + break + + if route_to_peer is None: + cls._random.shuffle(candidates) + return tuple(candidates) diff --git a/app/business/semantic_retrieval/AGENTS.md b/inkcre_core/business/semantic_retrieval/AGENTS.md similarity index 100% rename from app/business/semantic_retrieval/AGENTS.md rename to inkcre_core/business/semantic_retrieval/AGENTS.md diff --git a/inkcre_core/business/semantic_retrieval/__init__.py b/inkcre_core/business/semantic_retrieval/__init__.py new file mode 100644 index 00000000..c6a14424 --- /dev/null +++ b/inkcre_core/business/semantic_retrieval/__init__.py @@ -0,0 +1,35 @@ +"""Profile-scoped local semantic retrieval and embedding maintenance.""" + +from .main import ( + EmbeddingProfileNotFoundError, + InvalidSemanticVectorError, + SEMANTIC_RETRIEVAL_CONFIG_KEY, + SEMANTIC_RETRIEVAL_CONFIG_SCHEMA, + SEMANTIC_RETRIEVAL_CAPABILITY, + SemanticRetrievalDelegationError, + SemanticRetrievalError, + SemanticRetrievalManager, + SemanticRetrievalNotConfiguredError, +) +from .job import ( + SEMANTIC_MAINTAIN_JOB_TYPE, + SEMANTIC_REBUILD_JOB_TYPE, + SemanticMaintainJobHandler, + SemanticRebuildJobHandler, +) + +__all__ = [ + "EmbeddingProfileNotFoundError", + "InvalidSemanticVectorError", + "SEMANTIC_RETRIEVAL_CONFIG_KEY", + "SEMANTIC_RETRIEVAL_CONFIG_SCHEMA", + "SEMANTIC_RETRIEVAL_CAPABILITY", + "SemanticRetrievalDelegationError", + "SemanticRetrievalError", + "SemanticRetrievalManager", + "SemanticRetrievalNotConfiguredError", + "SEMANTIC_MAINTAIN_JOB_TYPE", + "SEMANTIC_REBUILD_JOB_TYPE", + "SemanticMaintainJobHandler", + "SemanticRebuildJobHandler", +] diff --git a/inkcre_core/business/semantic_retrieval/job.py b/inkcre_core/business/semantic_retrieval/job.py new file mode 100644 index 00000000..2c297013 --- /dev/null +++ b/inkcre_core/business/semantic_retrieval/job.py @@ -0,0 +1,59 @@ +"""Exact semantic embedding maintenance commands hosted by Jobs.""" + +from inkcre_core.business.job import JobHandler +from inkcre_core.schemas.job import JobModel +from inkcre_core.schemas.semantic_retrieval import EmbeddingMaintenanceJobParameters + +from .main import SemanticRetrievalManager + + +SEMANTIC_MAINTAIN_JOB_TYPE = "core.semantic_retrieval.maintain.v1" +SEMANTIC_REBUILD_JOB_TYPE = "core.semantic_retrieval.rebuild.v1" + + +class SemanticMaintainJobHandler( + JobHandler[EmbeddingMaintenanceJobParameters], + job_type=SEMANTIC_MAINTAIN_JOB_TYPE, + description="Maintain missing or stale semantic embedding records.", + parameters_model=EmbeddingMaintenanceJobParameters, + default_timeout_seconds=900, +): + @classmethod + async def can_handle(cls, parameters: EmbeddingMaintenanceJobParameters) -> bool: + return await SemanticRetrievalManager.can_maintain(parameters.profile) + + @classmethod + async def handle( + cls, + job: JobModel, + parameters: EmbeddingMaintenanceJobParameters, + ) -> None: + report = await SemanticRetrievalManager.maintain( + parameters.profile, + parameters.options, + ) + job.state = report.model_dump(mode="json") + + +class SemanticRebuildJobHandler( + JobHandler[EmbeddingMaintenanceJobParameters], + job_type=SEMANTIC_REBUILD_JOB_TYPE, + description="Rebuild semantic embedding records present before invocation.", + parameters_model=EmbeddingMaintenanceJobParameters, + default_timeout_seconds=1800, +): + @classmethod + async def can_handle(cls, parameters: EmbeddingMaintenanceJobParameters) -> bool: + return await SemanticRetrievalManager.can_maintain(parameters.profile) + + @classmethod + async def handle( + cls, + job: JobModel, + parameters: EmbeddingMaintenanceJobParameters, + ) -> None: + report = await SemanticRetrievalManager.rebuild( + parameters.profile, + parameters.options, + ) + job.state = report.model_dump(mode="json") diff --git a/inkcre_core/business/semantic_retrieval/main.py b/inkcre_core/business/semantic_retrieval/main.py new file mode 100644 index 00000000..aa3739c0 --- /dev/null +++ b/inkcre_core/business/semantic_retrieval/main.py @@ -0,0 +1,551 @@ +"""Profile-scoped embedding maintenance and exact semantic retrieval.""" + +from __future__ import annotations + +from dataclasses import dataclass +import datetime +import logging +import math +import typing + +import pydantic + +from inkcre_core.business.ai import AIExecutionRequirement, AIManager +from inkcre_core.business.deployment_config import ( + DeploymentConfigManager, + DeploymentConfigService, +) +from inkcre_core.business.info_base.services import RelationService +from inkcre_core.business.info_base.resolver import ( + ResolverManager, + UnknownResolverError, + UnsupportedResolverCapability, +) +from inkcre_core.business.peer import PeerManager +from inkcre_core.persistence.semantic_retrieval.uow import semantic_retrieval_uow +from inkcre_core.schemas.ai import ( + BlockEmbeddingModel, + EmbeddingProfileID, + EmbeddingProfileModel, + RelationEmbeddingModel, +) +from inkcre_core.schemas.info_base.block import BlockModel +from inkcre_core.schemas.info_base.relation import RelationModel +from inkcre_core.schemas.peer import PeerProtocolRequest, PeerProtocolResponse, PeerRef +from inkcre_core.schemas.semantic_retrieval import ( + BlockSemanticRetrievalMatch, + EmbeddingMaintenanceDiagnostic, + EmbeddingMaintenanceOptions, + EmbeddingMaintenanceReport, + RelationSemanticRetrievalMatch, + SemanticEntityType, + SemanticRetrievalConfig, + SemanticRetrievalMatch, + SemanticRetrievalRequest, + SemanticRetrievalResult, + VectorRetrievalOptions, +) + + +logger = logging.getLogger(__name__) + +SEMANTIC_RETRIEVAL_CONFIG_KEY = "semantic_retrieval" +SEMANTIC_RETRIEVAL_CONFIG_SCHEMA = "core.semantic_retrieval.config.v1" +SEMANTIC_RETRIEVAL_CAPABILITY = "core.semantic_retrieval.v1" + + +class SemanticRetrievalError(RuntimeError): + pass + + +class SemanticRetrievalNotConfiguredError(SemanticRetrievalError): + pass + + +class EmbeddingProfileNotFoundError(SemanticRetrievalError): + pass + + +class InvalidSemanticVectorError(SemanticRetrievalError): + pass + + +class SemanticRetrievalDelegationError(SemanticRetrievalError): + pass + + +class _ProjectionUnavailable(Exception): + pass + + +@dataclass(frozen=True) +class _Candidate: + entity_type: SemanticEntityType + entity_id: int + entity: BlockModel | RelationModel + text: str + + +@dataclass(frozen=True) +class _CandidatePage: + entities: tuple[BlockModel | RelationModel, ...] + next_cursor: int + exhausted: bool + + +@dataclass +class _ReportBuilder: + profile: EmbeddingProfileID + diagnostic_limit: int + embedded: int = 0 + unavailable: int = 0 + failed: int = 0 + diagnostics: list[EmbeddingMaintenanceDiagnostic] | None = None + + def __post_init__(self) -> None: + self.diagnostics = [] + + def record( + self, + entity_type: SemanticEntityType, + entity: int, + outcome: typing.Literal["unavailable", "failed"], + reason: str, + ) -> None: + if outcome == "unavailable": + self.unavailable += 1 + else: + self.failed += 1 + diagnostics = typing.cast(list[EmbeddingMaintenanceDiagnostic], self.diagnostics) + if len(diagnostics) < self.diagnostic_limit: + diagnostics.append( + EmbeddingMaintenanceDiagnostic( + entity_type=entity_type, + entity=entity, + outcome=outcome, + reason=reason, + ) + ) + + def build(self) -> EmbeddingMaintenanceReport: + return EmbeddingMaintenanceReport( + profile=self.profile, + embedded=self.embedded, + unavailable=self.unavailable, + failed=self.failed, + diagnostics=tuple(self.diagnostics or ()), + ) + + +DeploymentConfigManager.register_schema( + SEMANTIC_RETRIEVAL_CONFIG_SCHEMA, + SemanticRetrievalConfig, + keys=(SEMANTIC_RETRIEVAL_CONFIG_KEY,), +) + + +class SemanticRetrievalManager: + """Single use-domain owner for projection, records, ranking and defaults.""" + + @classmethod + async def get_profile( + cls, profile_id: EmbeddingProfileID + ) -> EmbeddingProfileModel | None: + async with semantic_retrieval_uow() as repository: + return await repository.get_profile(profile_id) + + @classmethod + async def list_profiles(cls, *, limit: int | None = None, cursor: int | None = None): + async with semantic_retrieval_uow() as repository: + return await repository.list_profiles(limit=limit, cursor=cursor) + + @classmethod + async def _configured_profile_id(cls) -> EmbeddingProfileID: + value = await DeploymentConfigService.get(SEMANTIC_RETRIEVAL_CONFIG_KEY) + if value is None: + raise SemanticRetrievalNotConfiguredError( + "Semantic retrieval default Profile is not configured" + ) + if not isinstance(value, SemanticRetrievalConfig): + raise TypeError("Semantic retrieval config registry returned the wrong model") + if value.default_profile is None: + raise SemanticRetrievalNotConfiguredError( + "Semantic retrieval default Profile is not configured" + ) + return value.default_profile + + @classmethod + async def _load_profile( + cls, + profile: EmbeddingProfileID | None, + ) -> EmbeddingProfileModel: + profile_id = await cls._configured_profile_id() if profile is None else profile + async with semantic_retrieval_uow() as repository: + result = await repository.get_profile(profile_id) + if result is None: + raise EmbeddingProfileNotFoundError(f"Embedding Profile {profile_id} does not exist") + return result + + @classmethod + async def can_maintain(cls, profile: EmbeddingProfileID | None = None) -> bool: + """Return static local Job eligibility without probing the provider.""" + try: + selected = await cls._load_profile(profile) + except (SemanticRetrievalNotConfiguredError, EmbeddingProfileNotFoundError): + return False + return await AIManager.can_execute( + selected.ai_model, + AIExecutionRequirement( + capability="embedding", + input_modalities=frozenset({"text"}), + output_modalities=frozenset({"vector"}), + ), + ) + + @classmethod + async def retrieve( + cls, + query: str, + profile: EmbeddingProfileID | None = None, + options: VectorRetrievalOptions | None = None, + *, + route_to_peer: PeerRef | None = None, + ) -> SemanticRetrievalResult: + """Execute locally unless the caller explicitly selects another Peer.""" + request = SemanticRetrievalRequest( + query=query, + profile=profile, + options=options or VectorRetrievalOptions(), + ) + if route_to_peer is None or route_to_peer == PeerManager.get_current_peer_ref(): + return await cls.retrieve_local( + request.query, + request.profile, + request.options, + ) + + payload = PeerProtocolRequest( + body=typing.cast( + typing.Any, + request.model_dump(mode="json"), + ) + ) + result = await PeerManager.delegate( + SEMANTIC_RETRIEVAL_CAPABILITY, + typing.cast(typing.Any, payload.model_dump(mode="json", exclude_unset=True)), + route_to_peer=route_to_peer, + ) + try: + response = PeerProtocolResponse.model_validate(result) + if response.status != 200 or "body" not in response.model_fields_set: + raise SemanticRetrievalDelegationError( + f"Semantic retrieval Peer returned HTTP {response.status}" + ) + return SemanticRetrievalResult.model_validate(response.body) + except pydantic.ValidationError as error: + raise SemanticRetrievalDelegationError( + "Semantic retrieval Peer returned an invalid response" + ) from error + + @classmethod + async def retrieve_local( + cls, + query: str, + profile: EmbeddingProfileID | None = None, + options: VectorRetrievalOptions | None = None, + ) -> SemanticRetrievalResult: + """Execute exact local retrieval without entering Peer delegation.""" + if not query.strip(): + raise ValueError("query must not be empty") + retrieval_options = options or VectorRetrievalOptions() + selected = await cls._load_profile(profile) + profile_id = cls._profile_id(selected) + query_vector = ( + await AIManager.embed( + selected.ai_model, + (query,), + selected.dimensions, + ) + )[0] + cls._require_non_zero_vector(query_vector) + + matches: list[SemanticRetrievalMatch] = [] + async with semantic_retrieval_uow() as repository: + if "block" in retrieval_options.entity_types: + rows = await repository.retrieve_blocks(selected, query_vector, retrieval_options) + matches.extend( + BlockSemanticRetrievalMatch(entity=block, score=cls._score(distance)) + for block, distance in rows + ) + if "relation" in retrieval_options.entity_types: + rows = await repository.retrieve_relations( + selected, query_vector, retrieval_options + ) + matches.extend( + RelationSemanticRetrievalMatch(entity=relation, score=cls._score(distance)) + for relation, distance in rows + ) + + matches.sort( + key=lambda match: ( + -match.score, + match.type, + typing.cast(int, match.entity.id), + ) + ) + return SemanticRetrievalResult( + profile=profile_id, + matches=tuple(matches[: retrieval_options.limit]), + ) + + @classmethod + async def maintain( + cls, + profile: EmbeddingProfileID | None = None, + options: EmbeddingMaintenanceOptions | None = None, + ) -> EmbeddingMaintenanceReport: + """Embed only missing/stale records and resume through durable successes.""" + selected = await cls._load_profile(profile) + return await cls._maintain( + selected, + options or EmbeddingMaintenanceOptions(), + rebuild_cutoff=None, + ) + + @classmethod + async def rebuild( + cls, + profile: EmbeddingProfileID | None = None, + options: EmbeddingMaintenanceOptions | None = None, + ) -> EmbeddingMaintenanceReport: + """Re-embed records that were present before this invocation began.""" + selected = await cls._load_profile(profile) + async with semantic_retrieval_uow() as repository: + cutoff = await repository.database_now() + return await cls._maintain( + selected, + options or EmbeddingMaintenanceOptions(), + rebuild_cutoff=cutoff, + ) + + @classmethod + async def maintain_default( + cls, + options: EmbeddingMaintenanceOptions, + ) -> EmbeddingMaintenanceReport | None: + """Scheduled no-op when this deployment has no selected default Profile.""" + try: + return await cls.maintain(options=options) + except SemanticRetrievalNotConfiguredError: + return None + + @classmethod + async def _maintain( + cls, + profile: EmbeddingProfileModel, + options: EmbeddingMaintenanceOptions, + *, + rebuild_cutoff: datetime.datetime | None, + ) -> EmbeddingMaintenanceReport: + profile_id = cls._profile_id(profile) + report = _ReportBuilder(profile_id, options.diagnostic_limit) + batch: list[_Candidate] = [] + + async def offer(entity_type: SemanticEntityType, entity) -> bool: + entity_id = typing.cast(int | None, entity.id) + if entity_id is None: + raise RuntimeError("Persisted semantic candidate is missing its ID") + try: + text = await cls._project(entity_type, entity) + except _ProjectionUnavailable as error: + report.record(entity_type, entity_id, "unavailable", str(error)) + return True + except Exception as error: + logger.exception( + "Semantic projection failed", + extra={"entity_type": entity_type, "entity": entity_id}, + ) + report.record(entity_type, entity_id, "failed", type(error).__name__) + return True + batch.append(_Candidate(entity_type, entity_id, entity, text)) + if ( + len(batch) >= options.batch_size + or report.embedded + len(batch) >= options.max_embeddings + ): + return await flush() + return True + + async def flush() -> bool: + if not batch: + return True + current = tuple(batch) + batch.clear() + try: + await cls._embed_and_upsert(profile, current) + except Exception as error: + logger.exception( + "Semantic embedding batch failed", + extra={"profile": profile_id, "batch_size": len(current)}, + ) + for candidate in current: + report.record( + candidate.entity_type, + candidate.entity_id, + "failed", + type(error).__name__, + ) + return False + report.embedded += len(current) + return report.embedded < options.max_embeddings + + for entity_type in typing.cast(tuple[SemanticEntityType, ...], ("block", "relation")): + cursor = 0 + while report.embedded + len(batch) < options.max_embeddings: + page = await cls._candidate_page( + profile, + entity_type, + cursor, + options.scan_page_size, + rebuild_cutoff, + ) + cursor = page.next_cursor + for entity in page.entities: + if not await offer(entity_type, entity): + return report.build() + if report.embedded >= options.max_embeddings: + return report.build() + if page.exhausted: + break + await flush() + return report.build() + + @classmethod + async def _project( + cls, + entity_type: SemanticEntityType, + entity: BlockModel | RelationModel, + ) -> str: + try: + if entity_type == "block": + text = await ResolverManager.get(typing.cast(BlockModel, entity)).get_text() + else: + text = await RelationService.get_text(typing.cast(RelationModel, entity)) + except UnknownResolverError as error: + raise _ProjectionUnavailable("unknown_resolver") from error + except UnsupportedResolverCapability as error: + raise _ProjectionUnavailable("unsupported_text") from error + if text is None or not text.strip(): + raise _ProjectionUnavailable("empty_projection") + return text + + @classmethod + async def _embed_and_upsert( + cls, + profile: EmbeddingProfileModel, + candidates: tuple[_Candidate, ...], + ) -> None: + vectors = await AIManager.embed( + profile.ai_model, + tuple(candidate.text for candidate in candidates), + profile.dimensions, + ) + for vector in vectors: + cls._require_non_zero_vector(vector) + + profile_id = cls._profile_id(profile) + blocks = [] + relations = [] + for candidate, vector in zip(candidates, vectors, strict=True): + row = { + "profile": profile_id, + candidate.entity_type: candidate.entity_id, + "embedding": vector, + } + (blocks if candidate.entity_type == "block" else relations).append(row) + async with semantic_retrieval_uow() as repository: + await repository.upsert(blocks, relations) + + @classmethod + async def _candidate_page( + cls, + profile: EmbeddingProfileModel, + entity_type: SemanticEntityType, + cursor: int, + page_size: int, + rebuild_cutoff: datetime.datetime | None, + ) -> _CandidatePage: + profile_id = cls._profile_id(profile) + async with semantic_retrieval_uow() as repository: + if entity_type == "block": + rows = await repository.candidate_blocks(profile_id, cursor, page_size) + entities = tuple( + block + for block, record in rows + if cls._block_requires_embedding(profile, block, record, rebuild_cutoff) + ) + else: + rows = await repository.candidate_relations(profile_id, cursor, page_size) + entities = tuple( + relation + for relation, record, from_updated, to_updated in rows + if cls._relation_requires_embedding( + profile, relation, record, (from_updated, to_updated), rebuild_cutoff + ) + ) + next_cursor = typing.cast(int, rows[-1][0].id) if rows else cursor + return _CandidatePage(entities, next_cursor, len(rows) < page_size) + + @staticmethod + def _block_requires_embedding( + profile: EmbeddingProfileModel, + block: BlockModel, + record: BlockEmbeddingModel | None, + rebuild_cutoff: datetime.datetime | None, + ) -> bool: + if record is None: + return True + if rebuild_cutoff is not None: + return record.updated_at < rebuild_cutoff + return ( + len(record.embedding) != profile.dimensions + or record.updated_at < profile.updated_at + or record.updated_at < block.updated_at + ) + + @staticmethod + def _relation_requires_embedding( + profile: EmbeddingProfileModel, + relation: RelationModel, + record: RelationEmbeddingModel | None, + endpoint_updated_at: tuple[datetime.datetime, datetime.datetime], + rebuild_cutoff: datetime.datetime | None, + ) -> bool: + if record is None: + return True + if rebuild_cutoff is not None: + return record.updated_at < rebuild_cutoff + return ( + len(record.embedding) != profile.dimensions + or record.updated_at < profile.updated_at + or record.updated_at < relation.updated_at + or record.updated_at < endpoint_updated_at[0] + or record.updated_at < endpoint_updated_at[1] + ) + + @staticmethod + def _profile_id(profile: EmbeddingProfileModel) -> EmbeddingProfileID: + if profile.id is None: + raise RuntimeError("Persisted Embedding Profile is missing its ID") + return profile.id + + @staticmethod + def _require_non_zero_vector(vector) -> None: + if not any(value != 0 for value in vector): + raise InvalidSemanticVectorError("Semantic vectors must be non-zero") + + @staticmethod + def _score(distance: float) -> float: + score = 1.0 - distance + if not math.isfinite(score): + raise InvalidSemanticVectorError("Cosine score must be finite") + return min(1.0, max(-1.0, score)) diff --git a/app/business/source/AGENTS.md b/inkcre_core/business/source/AGENTS.md similarity index 97% rename from app/business/source/AGENTS.md rename to inkcre_core/business/source/AGENTS.md index 40e6b608..361c5147 100644 --- a/app/business/source/AGENTS.md +++ b/inkcre_core/business/source/AGENTS.md @@ -1,6 +1,6 @@ # source/ Local Guide -本文件只描述 `app/business/source/` 的局部事实与变更风险。跨 subtree contract 看 +本文件只描述 `inkcre_core/business/source/` 的局部事实与变更风险。跨 subtree contract 看 [business-pipeline-and-authority.md](../../../docs/30-unit-tdd/business-pipeline-and-authority.md)。 ## 何时阅读 diff --git a/inkcre_core/business/source/__init__.py b/inkcre_core/business/source/__init__.py new file mode 100644 index 00000000..20b0d0c3 --- /dev/null +++ b/inkcre_core/business/source/__init__.py @@ -0,0 +1,24 @@ +from .main import SourceBase, SourceManager +from .config import ( + SOURCE_CONFIG_KEY, + SOURCE_CONFIG_SCHEMA_ID, + SourceDeploymentConfig, +) +from .job import ( + SOURCE_BACKFILL_JOB_TYPE, + SOURCE_COLLECT_JOB_TYPE, + SourceBackfillJobHandler, + SourceCollectJobHandler, +) + +__all__ = [ + "SourceBase", + "SourceManager", + "SOURCE_COLLECT_JOB_TYPE", + "SOURCE_BACKFILL_JOB_TYPE", + "SourceCollectJobHandler", + "SourceBackfillJobHandler", + "SOURCE_CONFIG_KEY", + "SOURCE_CONFIG_SCHEMA_ID", + "SourceDeploymentConfig", +] diff --git a/inkcre_core/business/source/config.py b/inkcre_core/business/source/config.py new file mode 100644 index 00000000..e5f8070b --- /dev/null +++ b/inkcre_core/business/source/config.py @@ -0,0 +1,46 @@ +"""Deployment-wide policy used by Source-owned content materialization.""" + +import typing + +import pydantic + +from inkcre_core.business.deployment_config import DeploymentConfigManager +from inkcre_core.business.info_base.storage import StorageManager, WritableStorage +from inkcre_core.schemas.info_base.storage import StorageID +from inkcre_core.schemas.source import SourceModel +from inkcre_core.persistence.source.uow import SourceUnitOfWork + + +SOURCE_CONFIG_KEY = "core.source" +SOURCE_CONFIG_SCHEMA_ID = "core.source.config.v1" +POSTGRESQL_BINARY_STORAGE_ID = -4 + + +class SourceDeploymentConfig(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid") + + default_storage: StorageID = POSTGRESQL_BINARY_STORAGE_ID + + +DeploymentConfigManager.register_schema( + SOURCE_CONFIG_SCHEMA_ID, SourceDeploymentConfig, keys=(SOURCE_CONFIG_KEY,) +) + + +async def resolve_writable_storage_async( + source: SourceModel, uow: SourceUnitOfWork +) -> WritableStorage: + """Resolve storage using the caller's Source/graph/config transaction.""" + storage_id = source.storage + if storage_id is None: + record = await uow.configuration.get(SOURCE_CONFIG_KEY) + persisted = None if record is None else DeploymentConfigManager._restore_record(record) + storage_id = ( + POSTGRESQL_BINARY_STORAGE_ID + if persisted is None + else typing.cast(SourceDeploymentConfig, persisted).default_storage + ) + storage = StorageManager.from_record(await uow.graph.storage.get(storage_id)) + if not isinstance(storage, WritableStorage): + raise ValueError(f"Storage {storage_id} is not writable") + return storage diff --git a/inkcre_core/business/source/job.py b/inkcre_core/business/source/job.py new file mode 100644 index 00000000..7b1bae06 --- /dev/null +++ b/inkcre_core/business/source/job.py @@ -0,0 +1,90 @@ +"""Exact Source command handlers hosted by the global Job runtime.""" + +from inkcre_core.business.job import JobHandler +from inkcre_core.persistence.job.uow import JobUnitOfWork +from inkcre_core.schemas.job import JobModel +from inkcre_core.schemas.source import ( + SourceBackfillParameters, + SourceCollectParameters, +) + +from inkcre_core.validation import input_path + +from .main import SourceManager, SourceNotFoundError + + +SOURCE_COLLECT_JOB_TYPE = "core.source.collect.v1" +SOURCE_BACKFILL_JOB_TYPE = "core.source.backfill.v1" + + +async def _source_type(parameters: SourceCollectParameters) -> str | None: + source = await SourceManager.get(parameters.source) + return None if source is None else source.type + + +class SourceCollectJobHandler( + JobHandler[SourceCollectParameters], + job_type=SOURCE_COLLECT_JOB_TYPE, + description="Run one ordinary collection command for a configured Source.", + parameters_model=SourceCollectParameters, + default_timeout_seconds=300, +): + @classmethod + async def normalize_parameters(cls, parameters: dict, uow: JobUnitOfWork) -> dict: + normalized = await super().normalize_parameters(parameters, uow) + source = await uow.sources.get(normalized["source"]) + if source is None: + raise SourceNotFoundError(f"Source {normalized['source']} does not exist") + with input_path("config"): + normalized["config"] = SourceManager.normalize_config( + source.type, + normalized["config"], + await uow.sources.get_type(source.type), + command="collect", + ) + return normalized + + @classmethod + async def can_handle(cls, parameters: SourceCollectParameters) -> bool: + source_type = await _source_type(parameters) + return source_type is not None and SourceManager.has_source_type(source_type) + + @classmethod + async def handle(cls, job: JobModel, parameters: SourceCollectParameters) -> None: + source = await SourceManager.get_source_ins(parameters.source) + config = source.validate_collect_config(parameters.config) + await source.collect(job, config) + + +class SourceBackfillJobHandler( + JobHandler[SourceBackfillParameters], + job_type=SOURCE_BACKFILL_JOB_TYPE, + description="Run one exact historical collection command for a configured Source.", + parameters_model=SourceBackfillParameters, + default_timeout_seconds=1800, +): + @classmethod + async def normalize_parameters(cls, parameters: dict, uow: JobUnitOfWork) -> dict: + normalized = await super().normalize_parameters(parameters, uow) + source = await uow.sources.get(normalized["source"]) + if source is None: + raise SourceNotFoundError(f"Source {normalized['source']} does not exist") + with input_path("config"): + normalized["config"] = SourceManager.normalize_config( + source.type, + normalized["config"], + await uow.sources.get_type(source.type), + command="backfill", + ) + return normalized + + @classmethod + async def can_handle(cls, parameters: SourceBackfillParameters) -> bool: + source_type = await _source_type(parameters) + return source_type is not None and SourceManager.supports_backfill(source_type) + + @classmethod + async def handle(cls, job: JobModel, parameters: SourceBackfillParameters) -> None: + source = await SourceManager.get_source_ins(parameters.source) + config = source.validate_backfill_config(parameters.config) + await source.backfill(job, config) diff --git a/inkcre_core/business/source/main.py b/inkcre_core/business/source/main.py new file mode 100644 index 00000000..0b67949b --- /dev/null +++ b/inkcre_core/business/source/main.py @@ -0,0 +1,320 @@ +from __future__ import annotations + +from inkcre_core.context import current_core + +import abc +import jsonschema # pyrefly: ignore[untyped-import] +import pydantic +import typing +from typing import Optional as Opt + +from inkcre_core.persistence.source.uow import SourceUnitOfWork, source_uow +from inkcre_core.schemas.info_base.block import BlockForm, BlockModel +from inkcre_core.schemas.job import JobModel +from inkcre_core.schemas.source import ( + SourceModel, + SourceID, + SourceTypesModel, + SourceUpdateForm, +) +from inkcre_core.validation import input_path +from .resolver import SOURCE_RESOLVER_ID, SourceContent + +ConfigTV = typing.TypeVar("ConfigTV", bound=pydantic.BaseModel) + + +class SourceNotFoundError(LookupError): + """A Source instance or persisted type does not exist.""" + + +class UnsupportedSourceCommandError(ValueError): + """The catalog explicitly does not support the requested collection mode.""" + + +class EmptySourceCommandConfig(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid") + + +class SourceBase(abc.ABC, typing.Generic[ConfigTV]): + """InkCre Source Base class. + + Configuration is loaded from the database. Instantiate a source when it is + used instead of caching configuration separately. + """ + + __configschema__: dict + """Source configuration JSON schema""" + __configcls__: type[ConfigTV] + __collectconfigcls__: type[pydantic.BaseModel] + __backfillconfigcls__: type[pydantic.BaseModel] | None + + def __init_subclass__( + cls, + config_cls: type[ConfigTV], + collect_config_cls: type[pydantic.BaseModel] = EmptySourceCommandConfig, + backfill_config_cls: type[pydantic.BaseModel] | None = None, + **kwargs, + ) -> None: + # ConfigTV is bound by the concrete source subclass. + cls.__configcls__ = config_cls # pyrefly: ignore[no-access] + cls.__configschema__ = config_cls.model_json_schema() + cls.__collectconfigcls__ = collect_config_cls # pyrefly: ignore[no-access] + cls.__backfillconfigcls__ = backfill_config_cls # pyrefly: ignore[no-access] + SourceManager.add_source_type(cls) + return super().__init_subclass__(**kwargs) + + def __init__(self, _id: SourceID) -> None: + self._id = _id + + @abc.abstractmethod + async def collect(self, job: JobModel, config: pydantic.BaseModel) -> None: + """Collect new data from the source. + + :param job: The collect job containing config and state. + + Notes: + - Should not surpress exceptions, raise it. + """ + + async def backfill(self, job: JobModel, config: pydantic.BaseModel) -> None: + del job, config + raise NotImplementedError(f"{self.__class__.__name__} does not support backfill") + + async def record(self, data: typing.Any) -> None: + """Record data passively (e.g., from webhook). + + :param data: The data to record from external source (e.g., webhook payload). + + Notes: + - Should not suppress exceptions, raise it. + - Used for passive collection methods like webhooks. + """ + raise NotImplementedError(f"{self.__class__.__name__} does not support passive record") + + async def get_config(self) -> ConfigTV: + """Read and validate current Source configuration in a short scope.""" + async with source_uow() as uow: + source = await uow.sources.get(self._id) + if source is None: + raise SourceNotFoundError(f"Source {self._id} does not exist") + return self.__configcls__.model_validate(source.config) + + def validate_collect_config(self, config: dict) -> pydantic.BaseModel: + return self.__collectconfigcls__.model_validate(config) + + def validate_backfill_config(self, config: dict) -> pydantic.BaseModel: + if self.__backfillconfigcls__ is None: + raise NotImplementedError(f"{self.__class__.__name__} does not support backfill") + return self.__backfillconfigcls__.model_validate(config) + + async def get_state(self) -> dict: + async with source_uow() as uow: + source = await uow.sources.get(self._id) + if source is None: + raise SourceNotFoundError(f"Source {self._id} does not exist") + return source.state or {} + + async def set_state(self, state: dict) -> None: + async with source_uow() as uow: + source = await uow.sources.get(self._id, lock=True) + if source is None: + raise SourceNotFoundError(f"Source {self._id} does not exist") + source.state = state + await uow.sources.save(source) + + +class SourceManager: + """ + + - Run collect method of all configured sources + - Add, remove and configure source instances + - Add, remove sources + """ + + _SOURCE_CLASSES: dict[str, type[SourceBase]] = {} + + @classmethod + def add_source_type(cls, source_cls: type[SourceBase]) -> None: + """Register a source type in memory without external side effects.""" + source_type = source_cls.__module__ + "." + source_cls.__qualname__ + cls._SOURCE_CLASSES[source_type] = source_cls + + @classmethod + async def sync_source_types_async( + cls, source_classes: dict[str, type[SourceBase]] | None = None + ) -> None: + registered = cls._SOURCE_CLASSES if source_classes is None else source_classes + rows = [ + dict( + id=source_type, + description=source_cls.__doc__ or "No description.", + config_schema=source_cls.__configschema__, + collect_config_schema=source_cls.__collectconfigcls__.model_json_schema(), + backfill_config_schema=( + None + if source_cls.__backfillconfigcls__ is None + else source_cls.__backfillconfigcls__.model_json_schema() + ), + ) + for source_type, source_cls in registered.items() + ] + async with source_uow() as uow: + await uow.sources.sync_types(rows) + + @classmethod + def has_source_type(cls, source_type: str) -> bool: + return source_type in cls._SOURCE_CLASSES + + @classmethod + def supports_backfill(cls, source_type: str) -> bool: + source_cls = cls._SOURCE_CLASSES.get(source_type) + return source_cls is not None and source_cls.__backfillconfigcls__ is not None + + @classmethod + async def get_source_ins(cls, source_id: SourceID) -> SourceBase: + ins = current_core().sources.get(source_id) + if ins is not None: + return ins + source = await cls.get(source_id) + if source is None: + raise SourceNotFoundError(f"Source {source_id} does not exist") + source_class = cls._SOURCE_CLASSES.get(source.type) + if source_class is None: + raise ValueError(f"Source class {source.type} not registered.") + ins = source_class(_id=source_id) + current_core().sources[source_id] = ins + return ins + + @classmethod + async def create( + cls, + type_: str, + nickname: Opt[str] = None, + config: dict | None = None, + storage: int | None = None, + ) -> SourceModel: + async with source_uow() as uow: + with input_path("config"): + normalized = cls.normalize_config( + type_, config or {}, await uow.sources.get_type(type_) + ) + source = SourceModel( + type=type_, nickname=nickname, config=normalized, storage=storage + ) + await uow.sources.save(source) + return source + + @classmethod + def normalize_config( + cls, + type_: str, + config: dict, + catalog: SourceTypesModel | None, + *, + command: typing.Literal["collect", "backfill"] | None = None, + ) -> dict: + """Accept catalog-only input without requiring local execution capability.""" + if catalog is None: + raise SourceNotFoundError(f"Source type {type_!r} does not exist") + source_class = cls._SOURCE_CLASSES.get(type_) + if command == "backfill": + schema = catalog.backfill_config_schema + model = source_class.__backfillconfigcls__ if source_class else None + elif command == "collect": + schema = catalog.collect_config_schema + model = source_class.__collectconfigcls__ if source_class else None + else: + schema = catalog.config_schema + model = source_class.__configcls__ if source_class else None # pyrefly: ignore[missing-attribute] + if schema is None: + raise UnsupportedSourceCommandError( + f"Source type {type_!r} does not support {command}" + ) + if model is not None: + return model.model_validate(config).model_dump(mode="json") + jsonschema.Draft202012Validator(schema).validate(config) + return config + + @classmethod + async def get(cls, source_id: SourceID) -> SourceModel | None: + async with source_uow() as uow: + return await uow.sources.get(source_id) + + @classmethod + async def get_type(cls, type_: str) -> SourceTypesModel | None: + async with source_uow() as uow: + return await uow.sources.get_type(type_) + + @classmethod + async def list_sources( + cls, *, limit: int | None = None, cursor: SourceID | None = None + ) -> tuple[list[SourceModel], SourceID | None]: + async with source_uow() as uow: + return await uow.sources.list_sources(limit=limit, cursor=cursor) + + @classmethod + async def list_types( + cls, *, limit: int | None = None, cursor: str | None = None + ) -> tuple[list[SourceTypesModel], str | None]: + async with source_uow() as uow: + return await uow.sources.list_types(limit=limit, cursor=cursor) + + @classmethod + async def update(cls, source_id: SourceID, form: SourceUpdateForm) -> SourceModel: + async with source_uow() as uow: + source = await uow.sources.get(source_id, lock=True) + if source is None: + raise SourceNotFoundError(f"Source {source_id} does not exist") + changes = form.model_dump(exclude_unset=True) + if "config" in changes: + with input_path("config"): + changes["config"] = cls.normalize_config( + source.type, changes["config"], await uow.sources.get_type(source.type) + ) + for field, value in changes.items(): + setattr(source, field, value) + await uow.sources.save(source) + if "nickname" in changes and source.block is not None: + await cls.ensure_block_async(source, uow) + return source + + @classmethod + async def delete(cls, source_id: SourceID) -> bool: + """Delete configuration only; collected graph and scheduled commands remain.""" + async with source_uow() as uow: + source = await uow.sources.get(source_id) + if source is None: + return False + await uow.sources.delete(source) + current_core().sources.pop(source_id, None) + return True + + @classmethod + async def ensure_block_async( + cls, source: SourceModel, uow: SourceUnitOfWork + ) -> BlockModel: + """Create/reuse a Source anchor in the caller's graph/state transaction.""" + if source.id is None: + raise ValueError("Source must be persisted before creating its anchor") + locked = await uow.sources.get(source.id, lock=True) + if locked is None: + raise SourceNotFoundError(f"Source {source.id} does not exist") + content = SourceContent( + id=source.id, type=locked.type, nickname=locked.nickname + ).model_dump_json() + if locked.block is None: + block = await uow.graph.blocks.create( + BlockForm(resolver=SOURCE_RESOLVER_ID, content=content) + ) + locked.block = block.id + await uow.sources.save(locked) + return block + block = await uow.graph.blocks.get(locked.block) + if block is None: + raise RuntimeError("Source anchor reference does not resolve") + if block.resolver != SOURCE_RESOLVER_ID or block.content != content: + block.resolver = SOURCE_RESOLVER_ID + block.storage = None + block.content = content + await uow.graph.blocks.save(block) + return block diff --git a/inkcre_core/business/source/resolver.py b/inkcre_core/business/source/resolver.py new file mode 100644 index 00000000..ec04cb62 --- /dev/null +++ b/inkcre_core/business/source/resolver.py @@ -0,0 +1,47 @@ +"""Use-time projection for lazy Source anchor Blocks.""" + +import pydantic + +from inkcre_core.business.info_base.resolver.label import format_label +from inkcre_core.business.info_base.resolver.main import Resolver, TextProjectionContext + + +SOURCE_RESOLVER_ID = "core.source.v1" + + +class SourceContent(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid") + + id: int + type: str + nickname: str | None = None + + +class SourceResolver(Resolver[SourceContent, str], rso_type=SOURCE_RESOLVER_ID): + async def _get_solved_content( + self, + *, + refresh: bool = False, + materialize_missing: bool = True, + ) -> SourceContent: + del materialize_missing + return SourceContent.model_validate_json(await self.get_raw_content(refresh=refresh)) + + async def get_text( + self, + *, + context: TextProjectionContext = "default", + refresh: bool = False, + materialize_missing: bool = True, + ) -> str: + del context + source = await self.get_solved_content( + refresh=refresh, + materialize_missing=materialize_missing, + ) + name = f" named {source.nickname}" if source.nickname else "" + return f"Source {source.id}{name} uses {source.type}." + + async def get_label(self, *, refresh: bool = False) -> str: + source = await self.get_solved_content(refresh=refresh, materialize_missing=False) + return format_label("source", source.nickname or source.type) diff --git a/inkcre_core/business/source/webpage.py b/inkcre_core/business/source/webpage.py new file mode 100644 index 00000000..74814835 --- /dev/null +++ b/inkcre_core/business/source/webpage.py @@ -0,0 +1 @@ +"""Webpage Source""" diff --git a/inkcre_core/configuration.py b/inkcre_core/configuration.py new file mode 100644 index 00000000..c09431af --- /dev/null +++ b/inkcre_core/configuration.py @@ -0,0 +1,59 @@ +"""Persistence-neutral mechanics for model-owned configuration values.""" + +from collections.abc import Mapping +import typing + +import pydantic + + +ConfigModelT = typing.TypeVar("ConfigModelT", bound=pydantic.BaseModel) + + +class ConfigContract(typing.Generic[ConfigModelT]): + """Apply one Pydantic model as a complete configuration contract. + + This abstraction deliberately has no persistence, registry, key, or live-update + behavior. Those consequences remain with the domain that owns the config. + """ + + def __init__(self, model: type[ConfigModelT]) -> None: + self.model = model + + def validate( + self, + value: ConfigModelT | Mapping[str, typing.Any], + ) -> ConfigModelT: + """Validate new input; pass an already constructed target model through. + + Passing a model back through Pydantic can rerun its model validators. + Typed callers have already crossed that boundary; persistence consumers may + still use one native model construction to restore nested Python types. + """ + if isinstance(value, self.model): + return value + return self.model.model_validate(value) + + def normalize( + self, + value: ConfigModelT | Mapping[str, typing.Any], + ) -> dict[str, typing.Any]: + """Return the complete value in JSON-compatible canonical form.""" + return self.validate(value).model_dump(mode="json") + + def prepare_patch( + self, + current: ConfigModelT | Mapping[str, typing.Any], + patch: Mapping[str, typing.Any], + ) -> ConfigModelT: + """Shallow-merge a patch and validate the resulting complete value.""" + candidate = ( + current.model_dump(mode="json") + if isinstance(current, pydantic.BaseModel) + else dict(current) + ) + candidate.update(patch) + return self.validate(candidate) + + def json_schema(self) -> dict[str, typing.Any]: + """Project the model's JSON Schema for UI or protocol discovery.""" + return self.model.model_json_schema() diff --git a/inkcre_core/context.py b/inkcre_core/context.py new file mode 100644 index 00000000..5013eb6a --- /dev/null +++ b/inkcre_core/context.py @@ -0,0 +1,103 @@ +"""Explicit database and execution scope for the SDK's domain operations.""" + +from __future__ import annotations + +from collections.abc import Generator +from contextlib import contextmanager +from contextvars import ContextVar +from dataclasses import dataclass, field +from typing import TYPE_CHECKING +from uuid import UUID, uuid4 +import asyncio + +from sqlalchemy.ext.asyncio import AsyncEngine, async_sessionmaker, create_async_engine + +if TYPE_CHECKING: + from .business.job import _Execution + from .business.peer.contracts import PeerInbound, PeerOutboundFactory + from .business.source.main import SourceBase + + +@dataclass(frozen=True) +class CoreOptions: + """Runtime values supplied by the consumer, independent of dotenv or HTTP servers.""" + + peer_id: UUID = field(default_factory=uuid4) + peer_name: str = "InKCre Python" + application_version: str = "development" + jwt_secret: str = "" + jwt_default_lifetime_seconds: int = 60 + peer_http_timeout_seconds: float = 30 + agent_debug: bool = False + + +_CURRENT: ContextVar[Core] = ContextVar("inkcre_core") + + +def current_core() -> Core: + """Require an explicit binding; there is no default deployment or global engine.""" + core = _CURRENT.get(None) + if core is None: + raise RuntimeError("Use 'with core.bind():' around InKCre operations") + if core._closed: + raise RuntimeError("This InKCre Core has been closed") + return core + + +class Core: + """Own SDK resources, or borrow an engine without taking its disposal authority. + + Binding is task-local and inherited by child tasks. Each operation still creates + its own short-lived Unit of Work; sessions are never stored in the context. + """ + + def __init__( + self, + database: str | AsyncEngine, + *, + options: CoreOptions | None = None, + pool_pre_ping: bool = False, + ) -> None: + self.options = options or CoreOptions() + self._owns_engine = isinstance(database, str) + self.engine = ( + create_async_engine(database, pool_pre_ping=pool_pre_ping) + if isinstance(database, str) + else database + ) + self.sessions = async_sessionmaker(self.engine, expire_on_commit=False) + self.sources: dict[int, SourceBase] = {} + self.inbounds: dict[str, PeerInbound] = {} + self.outbounds: dict[str, PeerOutboundFactory] = {} + self.active_jobs: dict[int, _Execution] = {} + self.pending_jobs: dict[int, asyncio.Task[bool]] = {} + self.accepting_jobs = False + self._closed = False + + @contextmanager + def bind(self) -> Generator[Core, None, None]: + """Bind this instance for an operation; nested bindings restore their caller.""" + if self._closed: + raise RuntimeError("This InKCre Core has been closed") + token = _CURRENT.set(self) + try: + yield self + finally: + _CURRENT.reset(token) + + async def aclose(self) -> None: + """Drain this instance's workers and dispose only an engine it created.""" + if self._closed: + return + with self.bind(): + if self.pending_jobs or self.active_jobs: + from .business.job import JobManager + + await JobManager.shutdown() + self.accepting_jobs = False + self.sources.clear() + self.inbounds.clear() + self.outbounds.clear() + if self._owns_engine: + await self.engine.dispose() + self._closed = True diff --git a/inkcre_core/database_contract/__init__.py b/inkcre_core/database_contract/__init__.py new file mode 100644 index 00000000..8943ac4e --- /dev/null +++ b/inkcre_core/database_contract/__init__.py @@ -0,0 +1,29 @@ +"""Executable peer database runtime contract.""" + +from .constants import ( + AUTHENTICATED_ROLE, + AUTHENTICATOR_ROLE, + CONTRACT_REVISION, + CORE_RUNTIME_ROLE, + DEVELOPMENT_PEER_ID, + INTERNAL_SCHEMA, + JWT_ALGORITHM, + JWT_AUDIENCE, + JWT_ISSUER, + JWT_MAX_LIFETIME_SECONDS, + PROTOCOL_SCHEMA, +) + +__all__ = [ + "AUTHENTICATED_ROLE", + "AUTHENTICATOR_ROLE", + "CONTRACT_REVISION", + "CORE_RUNTIME_ROLE", + "DEVELOPMENT_PEER_ID", + "INTERNAL_SCHEMA", + "JWT_ALGORITHM", + "JWT_AUDIENCE", + "JWT_ISSUER", + "JWT_MAX_LIFETIME_SECONDS", + "PROTOCOL_SCHEMA", +] diff --git a/inkcre_core/database_contract/constants.py b/inkcre_core/database_contract/constants.py new file mode 100644 index 00000000..c5dd790f --- /dev/null +++ b/inkcre_core/database_contract/constants.py @@ -0,0 +1,57 @@ +"""Stable names and values shared by migrations, runtimes, and consumers.""" + +import uuid + + +CONTRACT_FORMAT = 1 +CONTRACT_REVISION = "extension-distribution-open-v1" + +PROTOCOL_SCHEMA = "inkcre" +INTERNAL_SCHEMA = "inkcre_internal" + +AUTHENTICATED_ROLE = "authenticated" +AUTHENTICATOR_ROLE = "authenticator" +ANONYMOUS_ROLE = "anonymous" +CORE_RUNTIME_ROLE = "inkcre_core" + +JWT_ALGORITHM = "HS256" +JWT_ROLE = AUTHENTICATED_ROLE +JWT_ISSUER = "inkcre-peer" +JWT_AUDIENCE = "inkcre-api" +# Frozen input for admitted historical migrations; live admission has no lifetime cap. +JWT_MAX_LIFETIME_SECONDS = 24 * 60 * 60 +JWT_MINIMUM_SECRET_BYTES = 32 + +DEVELOPMENT_PEER_ID = uuid.UUID("00000000-0000-4000-8000-000000000001") +DEVELOPMENT_PEER_NAME = "client-web-development" + +DATABASE_ENVIRONMENTS = frozenset({"runtime", "development", "preview", "production"}) +RESET_CONFIRMATION = "reset-development-data" + +APPLICATION_TABLES = ( + "agents", + "ai_dialects", + "ai_models", + "ai_providers", + "block_embeddings", + "block_lexical_records", + "blocks", + "peers", + "configs", + "extensions", + "extension_peer_participations", + "embedding_profiles", + "crons", + "jobs", + "job_types", + "logs", + "relation_embeddings", + "relations", + "sink_types", + "sinks", + "sources", + "sources_types", + "storage_types", + "storage_blobs", + "storages", +) diff --git a/inkcre_core/database_contract/profile.py b/inkcre_core/database_contract/profile.py new file mode 100644 index 00000000..bfb56505 --- /dev/null +++ b/inkcre_core/database_contract/profile.py @@ -0,0 +1,386 @@ +"""Checked-in built-in catalog profile. + +The profile is data only. Reconciliation can consume it without importing FastAPI, +application settings, extension runtimes, or the scheduler. +""" + +from dataclasses import dataclass +from typing import Any + + +JsonObject = dict[str, Any] + + +@dataclass(frozen=True) +class TypeProfile: + id: str + description: str + config_schema: JsonObject + + +@dataclass(frozen=True) +class StorageTypeProfile: + id: str + description: str + config_schema: JsonObject + writable: bool + + +@dataclass(frozen=True) +class JobTypeProfile: + id: str + description: str + parameters_schema: JsonObject + default_timeout_seconds: int + + +@dataclass(frozen=True) +class StorageProfile: + id: int + type: str + nickname: str + config: JsonObject + + +def _object_schema( + title: str, + description: str, + properties: JsonObject, +) -> JsonObject: + return { + "description": description, + "properties": properties, + "title": title, + "type": "object", + } + + +def _positive_integer(default: int) -> JsonObject: + return {"default": default, "exclusiveMinimum": 0, "type": "integer"} + + +def _boolean(default: bool) -> JsonObject: + return {"default": default, "type": "boolean"} + + +HTTP_STORAGE_SCHEMA = _object_schema( + "HTTPStorageConfig", + "Configuration for HTTP storage.", + { + "follow_redirects": _boolean(True), + "max_response_bytes": _positive_integer(64 * 1024 * 1024), + "timeout": _positive_integer(30), + }, +) + +OPENAI_COMPATIBLE_DIALECT_SCHEMA = { + "additionalProperties": False, + "description": "Connection values for one OpenAI-compatible provider instance.", + "properties": { + "api_key": { + "minLength": 1, + "title": "Api Key", + "type": "string", + }, + "base_url": { + "anyOf": [ + {"minLength": 1, "type": "string"}, + {"type": "null"}, + ], + "default": None, + "title": "Base Url", + }, + }, + "required": ["api_key"], + "title": "OpenAICompatibleConfig", + "type": "object", +} + +BUILTIN_AI_DIALECTS = ( + TypeProfile( + "core.openai-compatible.v1", + "OpenAI-compatible embedding and chat protocol.", + OPENAI_COMPATIBLE_DIALECT_SCHEMA, + ), + TypeProfile( + "core.alibaba-model-studio.v1", + "Alibaba Model Studio OpenAI-compatible multimodal chat protocol.", + OPENAI_COMPATIBLE_DIALECT_SCHEMA, + ), +) + +BUILTIN_STORAGE_TYPES = ( + StorageTypeProfile( + "http", + "HTTP storage for bounded retrieval of opaque response bytes.", + HTTP_STORAGE_SCHEMA, + False, + ), +) + +BUILTIN_STORAGE_TYPES += ( + StorageTypeProfile( + "postgresql_binary", + "PostgreSQL storage for deployment-owned binary content.", + _object_schema( + "PostgreSQLBinaryStorageConfig", + "Configuration for PostgreSQL binary storage.", + {}, + ), + True, + ), +) + + +SOURCE_COLLECT_PARAMETERS_SCHEMA = { + "additionalProperties": False, + "description": "Parameters of the exact ordinary Source Job type.", + "properties": { + "config": { + "additionalProperties": True, + "title": "Config", + "type": "object", + }, + "source": {"title": "Source", "type": "integer"}, + }, + "required": ["source"], + "title": "SourceCollectParameters", + "type": "object", +} + +SOURCE_BACKFILL_PARAMETERS_SCHEMA = { + **SOURCE_COLLECT_PARAMETERS_SCHEMA, + "description": "Parameters of the exact historical Source Job type.", + "title": "SourceBackfillParameters", +} + +LEXICAL_MAINTENANCE_PARAMETERS_SCHEMA = { + "$defs": { + "LexicalMaintenanceOptions": { + "additionalProperties": False, + "properties": { + "diagnostic_limit": { + "default": 20, + "minimum": 0, + "title": "Diagnostic Limit", + "type": "integer", + }, + "max_records": { + "default": 100, + "minimum": 1, + "title": "Max Records", + "type": "integer", + }, + "scan_page_size": { + "default": 100, + "minimum": 1, + "title": "Scan Page Size", + "type": "integer", + }, + }, + "title": "LexicalMaintenanceOptions", + "type": "object", + } + }, + "additionalProperties": False, + "properties": { + "options": {"$ref": "#/$defs/LexicalMaintenanceOptions"}, + }, + "title": "LexicalMaintenanceJobParameters", + "type": "object", +} + +SEMANTIC_MAINTENANCE_PARAMETERS_SCHEMA = { + "$defs": { + "EmbeddingMaintenanceOptions": { + "additionalProperties": False, + "description": "Peer-local bounds for one resumable maintenance invocation.", + "properties": { + "batch_size": { + "default": 20, + "minimum": 1, + "title": "Batch Size", + "type": "integer", + }, + "diagnostic_limit": { + "default": 20, + "minimum": 0, + "title": "Diagnostic Limit", + "type": "integer", + }, + "max_embeddings": { + "default": 100, + "minimum": 1, + "title": "Max Embeddings", + "type": "integer", + }, + "scan_page_size": { + "default": 100, + "minimum": 1, + "title": "Scan Page Size", + "type": "integer", + }, + }, + "title": "EmbeddingMaintenanceOptions", + "type": "object", + } + }, + "additionalProperties": False, + "properties": { + "options": {"$ref": "#/$defs/EmbeddingMaintenanceOptions"}, + "profile": { + "anyOf": [{"type": "integer"}, {"type": "null"}], + "default": None, + "title": "Profile", + }, + }, + "title": "EmbeddingMaintenanceJobParameters", + "type": "object", +} + +MEDIA_INTERPRETATION_PARAMETERS_SCHEMA = { + "additionalProperties": False, + "properties": {}, + "title": "MediaInterpretationJobParameters", + "type": "object", +} + +AUTOMATIC_ORGANIZATION_PARAMETERS_SCHEMA = { + "additionalProperties": False, + "properties": { + "max_seeds": { + "default": 10, + "maximum": 100, + "minimum": 3, + "title": "Max Seeds", + "type": "integer", + } + }, + "title": "AutomaticOrganizationJobParameters", + "type": "object", +} + +BUILTIN_JOB_TYPES = ( + JobTypeProfile( + "core.source.collect.v1", + "Run one ordinary collection command for a configured Source.", + SOURCE_COLLECT_PARAMETERS_SCHEMA, + 300, + ), + JobTypeProfile( + "core.source.backfill.v1", + "Run one exact historical collection command for a configured Source.", + SOURCE_BACKFILL_PARAMETERS_SCHEMA, + 1800, + ), + JobTypeProfile( + "core.feature_retrieval.lexical.maintain.v1", + "Maintain missing or stale Block lexical projection records.", + LEXICAL_MAINTENANCE_PARAMETERS_SCHEMA, + 900, + ), + JobTypeProfile( + "core.feature_retrieval.lexical.rebuild.v1", + "Rebuild Block lexical records present before invocation.", + LEXICAL_MAINTENANCE_PARAMETERS_SCHEMA, + 1800, + ), + JobTypeProfile( + "core.semantic_retrieval.maintain.v1", + "Maintain missing or stale semantic embedding records.", + SEMANTIC_MAINTENANCE_PARAMETERS_SCHEMA, + 900, + ), + JobTypeProfile( + "core.semantic_retrieval.rebuild.v1", + "Rebuild semantic embedding records present before invocation.", + SEMANTIC_MAINTENANCE_PARAMETERS_SCHEMA, + 1800, + ), + JobTypeProfile( + "core.organization.media_interpretation.v1", + "Interpret missing image, audio, and video Blocks through configured Agents.", + MEDIA_INTERPRETATION_PARAMETERS_SCHEMA, + 1800, + ), + JobTypeProfile( + "core.organization.rumination.explicit.v1", + "Reconsider one explicitly selected Block through rumination.", + { + "additionalProperties": False, + "description": "Explicit request to reconsider one focal Block.", + "properties": {"block": {"title": "Block", "type": "integer"}}, + "required": ["block"], + "title": "RuminationRequest", + "type": "object", + }, + 1800, + ), + JobTypeProfile( + "core.organization.rumination.automatic.v1", + "Automatically reconsider bounded information seeds through rumination.", + AUTOMATIC_ORGANIZATION_PARAMETERS_SCHEMA, + 1800, + ), + JobTypeProfile( + "core.organization.supersession.automatic.v1", + "Automatically judge bounded scoped-supersession candidates.", + AUTOMATIC_ORGANIZATION_PARAMETERS_SCHEMA, + 1800, + ), + JobTypeProfile( + "core.organization.refinement.automatic.v1", + "Automatically judge bounded non-dominating refinement candidates.", + AUTOMATIC_ORGANIZATION_PARAMETERS_SCHEMA, + 1800, + ), + JobTypeProfile( + "core.organization.evidence-stance.automatic.v1", + "Automatically judge bounded attributable evidence stances.", + AUTOMATIC_ORGANIZATION_PARAMETERS_SCHEMA, + 1800, + ), + JobTypeProfile( + "core.organization.synthesis.automatic.v1", + "Automatically create bounded provenance-preserving syntheses.", + AUTOMATIC_ORGANIZATION_PARAMETERS_SCHEMA, + 1800, + ), + JobTypeProfile( + "core.organization.existing-referent-anchoring.automatic.v1", + "Automatically anchor bounded mentions to existing referents.", + AUTOMATIC_ORGANIZATION_PARAMETERS_SCHEMA, + 1800, + ), + JobTypeProfile( + "core.organization.duplicate-assertion.automatic.v1", + "Automatically judge bounded provenance-aware duplicate assertions.", + AUTOMATIC_ORGANIZATION_PARAMETERS_SCHEMA, + 1800, + ), + JobTypeProfile( + "core.sink.agent-query.v1", + "Answer one information need through an enabled Agent Query Sink.", + { + "additionalProperties": False, + "properties": { + "query": {"minLength": 1, "title": "Query", "type": "string"}, + "sink": {"title": "Sink", "type": "integer"}, + }, + "required": ["sink", "query"], + "title": "AgentQueryJobParameters", + "type": "object", + }, + 300, + ), +) + + +BUILTIN_STORAGES = ( + StorageProfile(-1, "http", "HTTP", {}), + StorageProfile(-4, "postgresql_binary", "PostgreSQL Binary", {}), +) + +BUILTIN_AI_DIALECTS_BY_ID = {item.id: item for item in BUILTIN_AI_DIALECTS} +BUILTIN_STORAGE_TYPES_BY_ID = {item.id: item for item in BUILTIN_STORAGE_TYPES} +BUILTIN_JOB_TYPES_BY_ID = {item.id: item for item in BUILTIN_JOB_TYPES} diff --git a/libs/AGENTS.md b/inkcre_core/libs/AGENTS.md similarity index 67% rename from libs/AGENTS.md rename to inkcre_core/libs/AGENTS.md index 23d94cef..d93f4319 100644 --- a/libs/AGENTS.md +++ b/inkcre_core/libs/AGENTS.md @@ -4,5 +4,5 @@ - Libraries must not acquire business-domain authority or import `app/business` policy. - Observability code may transport and format telemetry; it must not become the authority for application state. -- AI contracts belong in `app/schemas/ai/` and execution in `app/business/ai/`; do not reintroduce a process-global AI client under `libs/`. +- AI contracts belong in `inkcre_core/schemas/ai/` and execution in `inkcre_core/business/ai/`; do not reintroduce a process-global AI client under `libs/`. - Required check: run the affected library tests and import-boundary checks. diff --git a/inkcre_core/libs/obsrv/__init__.py b/inkcre_core/libs/obsrv/__init__.py new file mode 100644 index 00000000..39a4f499 --- /dev/null +++ b/inkcre_core/libs/obsrv/__init__.py @@ -0,0 +1,5 @@ +"""Observability package.""" + +__all__ = ["get_logger", "setup_obsrv"] + +from .main import get_logger, setup_obsrv diff --git a/inkcre_core/libs/obsrv/log_handler_logtail.py b/inkcre_core/libs/obsrv/log_handler_logtail.py new file mode 100644 index 00000000..61a93592 --- /dev/null +++ b/inkcre_core/libs/obsrv/log_handler_logtail.py @@ -0,0 +1,29 @@ +"""Logtail logging handler.""" + +import logging +from typing import Optional + +from logtail import LogtailHandler as RealLogtailHandler + + +class LogtailHandler: + """Wrapper for Logtail handler.""" + + def __init__(self, source_token: Optional[str] = None, host: Optional[str] = None): + self.source_token = source_token + self.host = host + self.handler = None + + def get_handler(self) -> Optional[logging.Handler]: + """Get the logtail handler if available.""" + if not self.source_token: + return None + if self.handler is None: + try: + self.handler = RealLogtailHandler(source_token=self.source_token) + if self.host: + self.handler.host = self.host + self.handler.setLevel(logging.INFO) + except Exception: + return None + return self.handler diff --git a/inkcre_core/libs/obsrv/log_handler_postgresql.py b/inkcre_core/libs/obsrv/log_handler_postgresql.py new file mode 100644 index 00000000..5b8d0d55 --- /dev/null +++ b/inkcre_core/libs/obsrv/log_handler_postgresql.py @@ -0,0 +1,75 @@ +"""Bounded bridge from synchronous logging calls to async PostgreSQL writes.""" + +import asyncio +import logging +from queue import Empty, Full, Queue +import sys + +from inkcre_core.libs.obsrv.log_record import ENABLE_LOG_BACKEND, LogModel, adapt_log_record + + +class PostgreSQLHandler(logging.Handler): + """Capture caller context immediately; persist only while lifespan is running.""" + + def __init__(self, level: int): + super().__init__(level) + self._queue: Queue[LogModel] = Queue(maxsize=1024) + self._task: asyncio.Task[None] | None = None + self._accepting = False + self.dropped = 0 + + def start(self) -> None: + if self._task is not None: + raise RuntimeError("PostgreSQL log writer is already running") + self._accepting = True + self._task = asyncio.create_task(self._write(), name="postgresql-log-writer") + + def emit(self, record: logging.LogRecord) -> None: + if not self._accepting or not ENABLE_LOG_BACKEND.get(): + return + try: + self._queue.put_nowait(adapt_log_record(record)) + except Full: + self.dropped += 1 + if self.dropped == 1: + print("PostgreSQL log queue full; dropping backend records", file=sys.stderr) + except Exception: + self.handleError(record) + + async def _write(self) -> None: + from inkcre_core.persistence.observability.uow import write_logs + + while self._accepting or not self._queue.empty(): + batch: list[LogModel] = [] + for _ in range(100): + try: + batch.append(self._queue.get_nowait()) + except Empty: + break + if not batch: + await asyncio.sleep(0.1) + continue + try: + await write_logs(batch) + except Exception as error: + # Report through stderr, never recursively through the failed backend. + print(f"PostgreSQL log batch failed: {type(error).__name__}", file=sys.stderr) + + async def aclose(self) -> None: + # Handler.handle() holds this lock even when emit originates on a worker thread. + self.acquire() + try: + self._accepting = False + finally: + self.release() + if self._task is not None: + try: + async with asyncio.timeout(5): + await self._task + except TimeoutError: + print("PostgreSQL log drain timed out; pending records lost", file=sys.stderr) + finally: + self._task = None + while not self._queue.empty(): + self._queue.get_nowait() + super().close() diff --git a/inkcre_core/libs/obsrv/log_record.py b/inkcre_core/libs/obsrv/log_record.py new file mode 100644 index 00000000..f0a1e869 --- /dev/null +++ b/inkcre_core/libs/obsrv/log_record.py @@ -0,0 +1,119 @@ +__all__ = ["LogModel", "adapt_log_record", "TRACE_ID", "SPAN_ID", "ENABLE_LOG_BACKEND"] + +from contextvars import ContextVar +import logging +import datetime +from typing import Mapping, Any, Dict, Optional as Opt +import sqlalchemy as sa +from sqlalchemy.dialects.postgresql import JSONB +import sqlmodel + +from inkcre_core.database_contract.constants import PROTOCOL_SCHEMA + + +TRACE_ID: ContextVar[str | None] = ContextVar("TRACE_ID", default=None) +SPAN_ID: ContextVar[str | None] = ContextVar("SPAN_ID", default=None) +ENABLE_LOG_BACKEND: ContextVar[bool] = ContextVar("ENABLE_LOG_BACKEND", default=False) +"""Whether the log backend emit the logs or not""" + + +LOGGING_TO_OTEL: Mapping[int, int] = { + 10: 5, # DEBUG + 20: 9, # INFO + 30: 13, # WARNING + 40: 17, # ERROR + 50: 21, # CRITICAL +} + + +def logging_level_to_otel(levelno: int) -> int: + return LOGGING_TO_OTEL.get(levelno, 9) + + +class LogModel(sqlmodel.SQLModel, table=True): + """Log model""" + + __tablename__ = "logs" # type: ignore + __table_args__ = {"schema": PROTOCOL_SCHEMA} + + id: Opt[int] = sqlmodel.Field( + default=None, + sa_column=sa.Column(sa.BigInteger, primary_key=True, autoincrement=True), + ) + timestamp: datetime.datetime = sqlmodel.Field( + default_factory=datetime.datetime.now, + sa_column=sa.Column( + sa.TIMESTAMP(timezone=True), + server_default=sa.text("CURRENT_TIMESTAMP"), + nullable=False, + ), + ) + severity_number: int = sqlmodel.Field(sa_type=sa.SmallInteger) + severity_text: str = sqlmodel.Field(sa_column=sa.Column(sa.Text, nullable=False)) + body: str = sqlmodel.Field(sa_column=sa.Column(sa.Text, nullable=False)) + trace_id: Opt[str] = sqlmodel.Field( + default=None, + sa_column=sa.Column(sa.Text, nullable=True), + ) + span_id: Opt[str] = sqlmodel.Field( + default=None, + sa_column=sa.Column(sa.Text, nullable=True), + ) + attributes: Dict[str, Any] = sqlmodel.Field( + sa_type=JSONB, + default_factory=dict, + sa_column_kwargs={"server_default": sa.text("'{}'::jsonb")}, + ) + + +def adapt_log_record( + record: logging.LogRecord, + # *, + # service_name: str, + # service_version: str | None = None, + # environment: str | None = None, +) -> LogModel: + attributes = { + k: v + for k, v in record.__dict__.items() + if k + not in { + "name", + "msg", + "message", + "args", + "levelname", + "levelno", + "pathname", + "filename", + "module", + "exc_info", + "exc_text", + "stack_info", + "lineno", + "funcName", + "created", + "msecs", + "relativeCreated", + "thread", + "threadName", + "processName", + "process", + "taskName", + "asctime", + } + and not k.startswith("_") + } + + return LogModel( + timestamp=datetime.datetime.fromtimestamp(record.created), + severity_number=logging_level_to_otel(record.levelno), + severity_text=record.levelname, + body=record.getMessage(), + trace_id=TRACE_ID.get(), + span_id=SPAN_ID.get(), + attributes=attributes, + # service_name=service_name, + # service_version=service_version, + # environment=environment, + ) diff --git a/inkcre_core/libs/obsrv/main.py b/inkcre_core/libs/obsrv/main.py new file mode 100644 index 00000000..72f5b0c3 --- /dev/null +++ b/inkcre_core/libs/obsrv/main.py @@ -0,0 +1,85 @@ +"""Main observability setup.""" + +import logging +import sys +from .setting import ObsrvSetting + +__all__ = ["get_logger"] + +LOGGER = logging.getLogger("inkcre") + + +def setup_obsrv(config: ObsrvSetting) -> logging.Logger: + """Setup observability components.""" + + # Create logger + global LOGGER + LOGGER = logging.getLogger("inkcre") + LOGGER.setLevel(logging.DEBUG) + + # Clear any existing handlers + LOGGER.handlers.clear() + + # Console handler (always enabled) + console_handler = logging.StreamHandler(sys.stdout) + console_handler.setLevel(logging.INFO) + console_formatter = logging.Formatter( + "%(asctime)s - %(name)s - %(levelname)s - %(message)s" + ) + console_handler.setFormatter(console_formatter) + LOGGER.addHandler(console_handler) + + # Add backend-specific handlers + backend = config.logging_backend + if backend == "logtail": + from .log_handler_logtail import LogtailHandler + + logtail_wrapper = LogtailHandler( + source_token=config.logtail_source_token, + host=config.logtail_host, + ) + handler = logtail_wrapper.get_handler() + if handler: + LOGGER.addHandler(handler) + LOGGER.info("Logtail logging enabled") + else: + LOGGER.warning("Logtail logging configured but not available") + elif backend == "postgresql": + from .log_handler_postgresql import PostgreSQLHandler + + pg_handler = PostgreSQLHandler(level=config.logging_backend_level) + LOGGER.addHandler(pg_handler) + LOGGER.info("PostgreSQL logging enabled") + elif backend not in (None, "", "none"): + LOGGER.warning(f"Unknown logging backend: {backend}") + + return LOGGER + + +def get_logger() -> logging.Logger: + """Get the application logger instance. + + Returns: + Logger instance for the application + """ + global LOGGER + return LOGGER + + +def start_obsrv() -> None: + from .log_handler_postgresql import PostgreSQLHandler + + for handler in LOGGER.handlers: + if isinstance(handler, PostgreSQLHandler): + handler.start() + + +async def close_obsrv() -> None: + from .log_handler_postgresql import PostgreSQLHandler + from .telemetry import close_telemetry + + await close_telemetry() + + for handler in LOGGER.handlers: + if isinstance(handler, PostgreSQLHandler): + await handler.aclose() diff --git a/inkcre_core/libs/obsrv/propagation.py b/inkcre_core/libs/obsrv/propagation.py new file mode 100644 index 00000000..5db0c9c3 --- /dev/null +++ b/inkcre_core/libs/obsrv/propagation.py @@ -0,0 +1,52 @@ +"""Standard trace propagation independent of application initialization.""" + +from collections.abc import Mapping +from opentelemetry.context import Context +from opentelemetry.trace import Link +from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator +from opentelemetry import trace +from .telemetry import emit_event, is_enabled + +PROPAGATOR = TraceContextTextMapPropagator() + + +def inject_context() -> dict[str, str]: + """Inject only standard Trace Context when this runtime explicitly enabled telemetry.""" + carrier: dict[str, str] = {} + if is_enabled(): + PROPAGATOR.inject(carrier) + return carrier + + +def submission_context() -> dict[str, str | None]: + carrier = inject_context() + state = carrier.get("tracestate") + if state is not None and len(state.encode("utf-8")) > 512: + state = None + emit_event("telemetry.carrier.dropped", {"reason": "tracestate_capacity"}) + return { + "submission_traceparent": carrier.get("traceparent"), + "submission_tracestate": state, + } + + +def extract_context(carrier: Mapping[str, str]) -> Context: + """Use the SDK parser within the application's bounded, two-header carrier.""" + if not is_enabled(): + return Context() + bounded = { + key: value + for key in ("traceparent", "tracestate") + if (value := carrier.get(key)) is not None and len(value.encode("utf-8")) <= 512 + } + return PROPAGATOR.extract(bounded, context=Context()) + + +def submission_links(parent: str | None, state: str | None) -> tuple[Link, ...]: + carrier = {} + if parent is not None: + carrier["traceparent"] = parent + if state is not None: + carrier["tracestate"] = state + context = trace.get_current_span(extract_context(carrier)).get_span_context() + return (Link(context),) if context.is_valid else () diff --git a/inkcre_core/libs/obsrv/setting.py b/inkcre_core/libs/obsrv/setting.py new file mode 100644 index 00000000..8e3a3346 --- /dev/null +++ b/inkcre_core/libs/obsrv/setting.py @@ -0,0 +1,35 @@ +"""Observability settings.""" + +from typing import Optional + +from pydantic import Field +from pydantic_settings import BaseSettings + + +class ObsrvSetting(BaseSettings): + """Observability settings.""" + + telemetry_enabled: bool = Field( + default=False, + description="Enable optional metadata-only OpenTelemetry export for this Peer.", + ) + + agent_debug: bool = Field( + default=False, + description="Record Agent inputs, tool contracts and execution events for development.", + ) + + logging_backend: Optional[str] = Field( + default="postgresql", + description="Logging backend to use (e.g., 'logtail', 'postgresql')", + ) + logging_backend_level: int = Field( + default=20, + description="Minimum log level for sending logs to the backend", + ) + logtail_source_token: Optional[str] = Field( + default=None, description="Better Stack (Logtail) source token for remote logging" + ) + logtail_host: Optional[str] = Field( + default=None, description="Better Stack (Logtail) host URL" + ) diff --git a/inkcre_core/libs/obsrv/telemetry.py b/inkcre_core/libs/obsrv/telemetry.py new file mode 100644 index 00000000..6343e9d9 --- /dev/null +++ b/inkcre_core/libs/obsrv/telemetry.py @@ -0,0 +1,488 @@ +"""Opt-in OTLP metadata transport; callers own allowed field names and value sources.""" + +import asyncio +from collections.abc import Callable, Generator, Mapping, Sequence +from contextlib import ExitStack, contextmanager +from dataclasses import dataclass, field +from functools import partial +import logging +import math +import os +import sys +import time +from typing import TYPE_CHECKING, Literal +from urllib.parse import urlsplit +import uuid + +from opentelemetry.context import Context +from opentelemetry.trace import INVALID_SPAN, Link, Span, SpanKind, StatusCode, use_span +from opentelemetry.util.types import AttributeValue +from pydantic import SecretStr +from pydantic_settings import BaseSettings, SettingsConfigDict + +if TYPE_CHECKING: + from opentelemetry.sdk._logs import LoggerProvider + from opentelemetry.sdk.metrics import MeterProvider + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.metrics import Counter, Histogram + from opentelemetry.trace import Tracer + from opentelemetry._logs import Logger + + +class _ExportSettings(BaseSettings): + model_config = SettingsConfigDict(extra="ignore") + + otel_exporter_otlp_traces_endpoint: str = "" + otel_exporter_otlp_logs_endpoint: str = "" + otel_exporter_otlp_metrics_endpoint: str = "" + otel_exporter_otlp_headers: SecretStr = SecretStr("") + otel_exporter_otlp_traces_headers: SecretStr = SecretStr("") + otel_exporter_otlp_logs_headers: SecretStr = SecretStr("") + otel_exporter_otlp_metrics_headers: SecretStr = SecretStr("") + # Validate each signal separately, so a bad timeout does not disable siblings. + otel_exporter_otlp_timeout: str = "10" + otel_exporter_otlp_traces_timeout: str = "" + otel_exporter_otlp_logs_timeout: str = "" + otel_exporter_otlp_metrics_timeout: str = "" + + +@dataclass +class _Runtime: + connections: dict[str, tuple[str, dict[str, str], float]] = field(default_factory=dict) + traces: "TracerProvider | None" = None + logs: "LoggerProvider | None" = None + metrics: "MeterProvider | None" = None + tracer: "Tracer | None" = None + logger: "Logger | None" = None + operations: "Counter | None" = None + duration: "Histogram | None" = None + tokens: "Counter | None" = None + missing_usage: "Counter | None" = None + + +_runtime: _Runtime | None = None + + +def _diagnostic(reason: str) -> None: + # This sink never receives endpoints, headers, exceptions, or arbitrary log messages. + try: + print(f"OpenTelemetry: {reason}", file=sys.stderr) + except Exception: + # Diagnostics cannot make a broken stderr stream a business failure. + pass + + +class _SDKDiagnostics(logging.Handler): + def emit(self, record: logging.LogRecord) -> None: + # SDK errors may contain response bodies, URLs or malformed secret headers. + _diagnostic("SDK warning or export failure; check the configured OTLP receiver") + + +def _configure_diagnostics() -> None: + logger = logging.getLogger("opentelemetry") + if not any(isinstance(handler, _SDKDiagnostics) for handler in logger.handlers): + handler = _SDKDiagnostics(level=logging.WARNING) + logger.addHandler(handler) + logger.propagate = False + + +def _endpoint(value: str) -> str: + parsed = urlsplit(value) + if ( + parsed.scheme not in ("http", "https") + or not parsed.hostname + or not parsed.path + or parsed.username is not None + or parsed.password is not None + or parsed.query + or parsed.fragment + or any(character.isspace() for character in value) + ): + raise ValueError("invalid OTLP URL") + # Accessing port performs urllib's range and syntax validation. + parsed.port + return value + + +def is_enabled() -> bool: + """Whether at least one metadata signal initialized for this runtime.""" + return _runtime is not None + + +def get_export_connection(signal: str) -> tuple[str, dict[str, str], float] | None: + """Return private endpoint, copied headers and timeout seconds for server forwarding.""" + connection = _runtime.connections.get(signal) if _runtime is not None else None + return ( + (connection[0], dict(connection[1]), connection[2]) if connection is not None else None + ) + + +def start_telemetry( + *, + enabled: bool, + service_version: str, + resource_attributes: Mapping[str, AttributeValue] | None = None, + endpoints: Mapping[str, str] | None = None, +) -> None: + """Initialize only configured signals; configuration failures never gate readiness.""" + global _runtime + if not enabled or _runtime is not None: + return + + _configure_diagnostics() + try: + config = _ExportSettings(_env_file=os.getenv("INKCRE_ENV_FILE", ".env") or None) + except Exception: + _diagnostic("invalid runtime export configuration; telemetry is disabled") + return + + from opentelemetry.exporter.otlp.proto.http import Compression + from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter + from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter + from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter + from opentelemetry.metrics import NoOpMeterProvider + from opentelemetry.sdk._logs import LoggerProvider, LogRecordLimits + from opentelemetry.sdk._logs.export import BatchLogRecordProcessor + from opentelemetry.sdk.metrics import MeterProvider, SimpleFixedSizeExemplarReservoir + from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader + from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View + from opentelemetry.sdk.resources import Resource + from opentelemetry.sdk.trace import SpanLimits, TracerProvider + from opentelemetry.sdk.trace.export import BatchSpanProcessor + from opentelemetry.sdk.trace.sampling import ALWAYS_ON + from opentelemetry.util.re import parse_env_headers + from requests.utils import check_header_validity # type: ignore[untyped-import] + + # Resource.create invokes environment detectors; direct construction is deliberate. + resource = Resource( + { + **(resource_attributes or {}), + "service.name": "core-py", + "service.version": service_version, + "service.instance.id": str(uuid.uuid4()), + } + ) + runtime = _Runtime() + no_metrics = NoOpMeterProvider() + for signal in ("metrics", "traces", "logs"): + endpoint = getattr(config, f"otel_exporter_otlp_{signal}_endpoint") or ( + endpoints or {} + ).get(signal, "") + if not endpoint: + continue + cleanup: Callable[[], None] | None = None + internal_metrics = runtime.metrics or no_metrics + try: + endpoint = _endpoint(endpoint) + timeout = float( + getattr(config, f"otel_exporter_otlp_{signal}_timeout") + or config.otel_exporter_otlp_timeout + or "10" + ) + if not math.isfinite(timeout) or not 0 < timeout <= 30: + raise ValueError("OTLP timeout must be finite, positive and at most 30 seconds") + raw_headers = ( + getattr(config, f"otel_exporter_otlp_{signal}_headers").get_secret_value() + or config.otel_exporter_otlp_headers.get_secret_value() + ) + headers = parse_env_headers(raw_headers, liberal=True) + # The SDK parser drops malformed entries. Do not silently accept partial auth. + if len(headers) != len([part for part in raw_headers.split(",") if part.strip()]): + raise ValueError("invalid OTLP headers") + for header in headers.items(): + check_header_validity(header) + + if signal == "traces": + exporter = OTLPSpanExporter( + endpoint=endpoint, + headers=headers, + timeout=timeout, + compression=Compression.NoCompression, + meter_provider=internal_metrics, + ) + cleanup = exporter.shutdown + traces = TracerProvider( + resource=resource, + sampler=ALWAYS_ON, + shutdown_on_exit=False, + span_limits=SpanLimits( + max_attributes=64, + max_events=16, + max_links=16, + max_attribute_length=256, + max_span_attributes=64, + max_event_attributes=64, + max_link_attributes=64, + max_span_attribute_length=256, + ), + meter_provider=internal_metrics, + ) + processor = BatchSpanProcessor( + exporter, + max_queue_size=256, + max_export_batch_size=256, + schedule_delay_millis=5000, + export_timeout_millis=timeout * 1000, + meter_provider=internal_metrics, + ) + cleanup = processor.shutdown + traces.add_span_processor(processor) + cleanup = traces.shutdown + tracer = traces.get_tracer("inkcre.metadata") + runtime.traces, runtime.tracer = traces, tracer + elif signal == "logs": + log_exporter = OTLPLogExporter( + endpoint=endpoint, + headers=headers, + timeout=timeout, + compression=Compression.NoCompression, + meter_provider=internal_metrics, + ) + cleanup = log_exporter.shutdown + logs = LoggerProvider( + resource=resource, + shutdown_on_exit=False, + meter_provider=internal_metrics, + log_record_limits=LogRecordLimits(max_attributes=64, max_attribute_length=256), + ) + log_processor = BatchLogRecordProcessor( + log_exporter, + max_queue_size=256, + max_export_batch_size=256, + schedule_delay_millis=5000, + export_timeout_millis=timeout * 1000, + meter_provider=internal_metrics, + ) + cleanup = log_processor.shutdown + logs.add_log_record_processor(log_processor) + cleanup = logs.shutdown + logger = logs.get_logger("inkcre.metadata") + runtime.logs, runtime.logger = logs, logger + else: + metric_exporter = OTLPMetricExporter( + endpoint=endpoint, + headers=headers, + timeout=timeout, + compression=Compression.NoCompression, + meter_provider=no_metrics, + ) + cleanup = metric_exporter.shutdown + reader = PeriodicExportingMetricReader( + metric_exporter, + export_interval_millis=30000, + export_timeout_millis=timeout * 1000, + ) + cleanup = partial(reader.shutdown, timeout_millis=(timeout + 2) * 1000) + metrics = MeterProvider( + resource=resource, + metric_readers=[reader], + shutdown_on_exit=False, + views=[ + # SDK defaults are too coarse for a duration measured in seconds. + View( + instrument_name="inkcre.operation.duration", + aggregation=ExplicitBucketHistogramAggregation( + boundaries=( + 0.005, + 0.01, + 0.025, + 0.05, + 0.1, + 0.25, + 0.5, + 1, + 2.5, + 5, + 10, + 30, + 60, + 300, + ) + ), + ), + View( + meter_name="opentelemetry-sdk", + attribute_keys={ + "otel.component.type", + "error.type", + "http.response.status_code", + }, + # An exemplar can otherwise retain attributes excluded by the View. + exemplar_reservoir_factory=lambda _: ( + lambda _config=None, **_kwargs: SimpleFixedSizeExemplarReservoir(size=0) + ), + ), + ], + ) + cleanup = partial(metrics.shutdown, timeout_millis=(timeout + 2) * 1000) + meter = metrics.get_meter("inkcre.metadata") + operations = meter.create_counter("inkcre.operation.count", unit="{operation}") + duration = meter.create_histogram("inkcre.operation.duration", unit="s") + tokens = meter.create_counter("inkcre.ai.token.usage", unit="{token}") + missing_usage = meter.create_counter("inkcre.ai.usage.missing", unit="{field}") + runtime.metrics = metrics + runtime.operations, runtime.duration = operations, duration + runtime.tokens, runtime.missing_usage = tokens, missing_usage + except Exception: + if cleanup is not None: + _shutdown(cleanup) + _diagnostic(f"{signal} configuration or initialization failed; signal is disabled") + else: + runtime.connections[signal] = (endpoint, dict(headers), timeout) + + if any((runtime.traces, runtime.logs, runtime.metrics)): + _runtime = runtime + else: + _diagnostic("no usable per-signal endpoint; telemetry is disabled") + + +def _shutdown(shutdown: Callable[[], None]) -> None: + try: + shutdown() + except Exception: + _diagnostic("signal shutdown failed; remaining telemetry may be lost") + + +async def close_telemetry() -> None: + """Stop new records and concurrently drain SDK queues outside the event loop. + + SDK trace/log shutdown waits up to 30 seconds for each worker, then metrics gets + its configured request timeout plus two seconds. These worker wait budgets are + not transport-wide hard deadlines; freezing or termination can lose records. + """ + global _runtime + runtime, _runtime = _runtime, None + if runtime is None: + return + shutdowns: list[Callable[[], None]] = [] + for provider in (runtime.traces, runtime.logs): + if provider is not None: + shutdowns.append(provider.shutdown) + # SDK 1.45 force_flush ignores its timeout; shutdown owns the bounded queue drain. + await asyncio.gather(*(asyncio.to_thread(_shutdown, shutdown) for shutdown in shutdowns)) + if runtime.metrics is not None: + # Collect final native processor/exporter metrics after their workers have drained. + await asyncio.to_thread( + _shutdown, + partial( + runtime.metrics.shutdown, + timeout_millis=(runtime.connections["metrics"][2] + 2) * 1000, + ), + ) + + +def _record(action: Callable[[], object]) -> None: + """Isolate one owned telemetry call; never wrap the caller's business block.""" + try: + action() + except Exception: + _diagnostic("recording failed; optional telemetry may be lost") + + +@dataclass +class Operation: + """One execution scope's result, independent of whether its Span is recorded. + + Business boundaries set outcome when they return a failure or handle cancellation. + Span methods remain available explicitly through span; Span status is not a metric. + Those standard SDK methods require the caller's admitted primitive metadata values; + the facade isolates its own SDK calls, not arbitrary caller code inside the scope. + """ + + span: Span + outcome: Literal["success", "error", "cancelled"] = "success" + + +@contextmanager +def operation( + name: str, + *, + attributes: Mapping[str, AttributeValue] | None = None, + context: Context | None = None, + links: Sequence[Link] | None = None, + kind: SpanKind = SpanKind.INTERNAL, +) -> Generator[Operation, None, None]: + """Measure a fixed operation name; callers supply only admitted metadata values. + + Exceptions and cancellation propagate unchanged. Their messages, stack traces and + status descriptions are never recorded by this context manager. + """ + runtime = _runtime + if runtime is None: + yield Operation(INVALID_SPAN) + return + + span = INVALID_SPAN + if runtime.tracer is not None: + try: + span = runtime.tracer.start_span( + name, + attributes=attributes, + context=context, + links=links, + kind=kind, + record_exception=False, + set_status_on_exception=False, + ) + except Exception: + _diagnostic("span creation failed; operation metrics remain available") + observation = Operation(span) + started = time.perf_counter() + activation = ExitStack() + _record( + lambda: activation.enter_context( + use_span(span, record_exception=False, set_status_on_exception=False) + ) + ) + try: + yield observation + except BaseException as error: + observation.outcome = ( + "cancelled" if isinstance(error, asyncio.CancelledError) else "error" + ) + _record(partial(span.set_attribute, "error.type", type(error).__name__)) + raise + finally: + # Exit the SDK context with no exception and end separately. This prevents SDK + # context cleanup or span processors from masking the caller's original error. + if observation.outcome != "success": + _record(partial(span.set_status, StatusCode.ERROR)) + _record(partial(span.set_attribute, "inkcre.outcome", observation.outcome)) + labels = {"operation": name, "outcome": observation.outcome} + if runtime.operations is not None: + _record(partial(runtime.operations.add, 1, labels)) + if runtime.duration is not None: + _record(partial(runtime.duration.record, time.perf_counter() - started, labels)) + _record(activation.close) + _record(span.end) + + +def emit_event(name: str, attributes: Mapping[str, AttributeValue]) -> None: + """Emit a fixed event name and admitted metadata without a Python logging bridge.""" + runtime = _runtime + if runtime is not None and runtime.logger is not None: + from opentelemetry._logs import SeverityNumber + + _record( + partial( + runtime.logger.emit, + body=name, + severity_number=SeverityNumber.INFO, + attributes=attributes, + ) + ) + + +def record_ai_usage( + operation: str, input_tokens: int | None, output_tokens: int | None +) -> None: + """Count known nonnegative provider usage and count missing fields separately.""" + runtime = _runtime + if runtime is None or runtime.tokens is None or runtime.missing_usage is None: + return + for kind, value in (("input", input_tokens), ("output", output_tokens)): + labels = {"operation": operation, "token.type": kind} + if isinstance(value, int) and not isinstance(value, bool) and value >= 0: + _record(partial(runtime.tokens.add, value, labels)) + else: + _record(partial(runtime.missing_usage.add, 1, labels)) diff --git a/inkcre_core/persistence/__init__.py b/inkcre_core/persistence/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/inkcre_core/persistence/agent/__init__.py b/inkcre_core/persistence/agent/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/inkcre_core/persistence/agent/repository.py b/inkcre_core/persistence/agent/repository.py new file mode 100644 index 00000000..2ac48f47 --- /dev/null +++ b/inkcre_core/persistence/agent/repository.py @@ -0,0 +1,49 @@ +"""Persisted Agent definitions; Thread runtime state has a separate owner.""" + +import sqlmodel +from sqlalchemy.ext.asyncio import AsyncSession +from inkcre_core.schemas.agent import AgentDefinitionModel, AgentForm + + +class AgentRepository: + def __init__(self, session: AsyncSession) -> None: + self._session = session + + async def get( + self, agent_id: int, *, for_update: bool = False + ) -> AgentDefinitionModel | None: + statement = sqlmodel.select(AgentDefinitionModel).where( + AgentDefinitionModel.id == agent_id + ) + if for_update: + statement = statement.with_for_update() + return (await self._session.scalars(statement)).one_or_none() + + async def list( + self, *, limit: int | None, cursor: int | None + ) -> tuple[list[AgentDefinitionModel], int | None]: + statement = sqlmodel.select(AgentDefinitionModel).order_by( + sqlmodel.col(AgentDefinitionModel.id) + ) + if cursor is not None: + statement = statement.where(sqlmodel.col(AgentDefinitionModel.id) > cursor) + if limit is not None: + statement = statement.limit(limit + 1) + rows = list((await self._session.scalars(statement)).all()) + more = limit is not None and len(rows) > limit + rows = rows[:limit] + return rows, rows[-1].id if more else None + + async def create(self, form: AgentForm) -> AgentDefinitionModel: + record = AgentDefinitionModel(**form.model_dump()) + self._session.add(record) + await self._session.flush() + return record + + async def save(self, record: AgentDefinitionModel) -> None: + self._session.add(record) + await self._session.flush() + await self._session.refresh(record) + + async def delete(self, record: AgentDefinitionModel) -> None: + await self._session.delete(record) diff --git a/inkcre_core/persistence/agent/uow.py b/inkcre_core/persistence/agent/uow.py new file mode 100644 index 00000000..a79040ec --- /dev/null +++ b/inkcre_core/persistence/agent/uow.py @@ -0,0 +1,13 @@ +"""Short transaction for persisted Agent definitions.""" + +from collections.abc import AsyncGenerator +from contextlib import asynccontextmanager + +from inkcre_core.context import current_core +from .repository import AgentRepository + + +@asynccontextmanager +async def agent_uow() -> AsyncGenerator[AgentRepository, None]: + async with current_core().sessions.begin() as session: + yield AgentRepository(session) diff --git a/inkcre_core/persistence/ai/__init__.py b/inkcre_core/persistence/ai/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/inkcre_core/persistence/ai/repository.py b/inkcre_core/persistence/ai/repository.py new file mode 100644 index 00000000..f1b0bfed --- /dev/null +++ b/inkcre_core/persistence/ai/repository.py @@ -0,0 +1,47 @@ +"""AI registry facts inside the caller's transaction.""" + +from collections.abc import Sequence +from typing import Any + +import sqlmodel +from sqlalchemy.dialects.postgresql import insert +from sqlalchemy.ext.asyncio import AsyncSession + +from inkcre_core.schemas.ai import AIDialectModel, AIModelModel, AIProviderModel + + +class AIRepository: + def __init__(self, session: AsyncSession) -> None: + self._session = session + + async def get_model(self, model_id: int) -> AIModelModel | None: + return await self._session.get(AIModelModel, model_id) + + async def get_provider(self, provider_id: int) -> AIProviderModel | None: + return await self._session.get(AIProviderModel, provider_id) + + async def list_models( + self, *, limit: int | None, cursor: int | None + ) -> tuple[list[AIModelModel], int | None]: + statement = sqlmodel.select(AIModelModel).order_by(sqlmodel.col(AIModelModel.id)) + if cursor is not None: + statement = statement.where(sqlmodel.col(AIModelModel.id) > cursor) + if limit is not None: + statement = statement.limit(limit + 1) + rows = list((await self._session.scalars(statement)).all()) + more = limit is not None and len(rows) > limit + rows = rows[:limit] + return rows, rows[-1].id if more else None + + async def sync_dialects(self, records: Sequence[dict[str, Any]]) -> None: + if not records: + return + statement = insert(AIDialectModel).values(records) + statement = statement.on_conflict_do_update( + index_elements=[AIDialectModel.id], + set_={ + "description": statement.excluded.description, + "config_schema": statement.excluded.config_schema, + }, + ) + await self._session.execute(statement) diff --git a/inkcre_core/persistence/ai/uow.py b/inkcre_core/persistence/ai/uow.py new file mode 100644 index 00000000..82b147c9 --- /dev/null +++ b/inkcre_core/persistence/ai/uow.py @@ -0,0 +1,13 @@ +"""Short database scope for AI facts; provider calls outlive this scope.""" + +from collections.abc import AsyncGenerator +from contextlib import asynccontextmanager + +from inkcre_core.context import current_core +from .repository import AIRepository + + +@asynccontextmanager +async def ai_uow() -> AsyncGenerator[AIRepository, None]: + async with current_core().sessions.begin() as session: + yield AIRepository(session) diff --git a/inkcre_core/persistence/cron/__init__.py b/inkcre_core/persistence/cron/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/inkcre_core/persistence/cron/repository.py b/inkcre_core/persistence/cron/repository.py new file mode 100644 index 00000000..e60ebfb5 --- /dev/null +++ b/inkcre_core/persistence/cron/repository.py @@ -0,0 +1,50 @@ +"""Locked Cron occurrences and database-clock observations.""" + +import datetime + +import sqlmodel +from sqlalchemy.ext.asyncio import AsyncSession + +from inkcre_core.schemas.cron import CronModel + + +class CronRepository: + def __init__(self, session: AsyncSession) -> None: + self._session = session + + async def get(self, cron_id: int, *, lock: bool = False, skip_locked: bool = False): + statement = sqlmodel.select(CronModel).where(CronModel.id == cron_id) + if lock: + statement = statement.with_for_update(skip_locked=skip_locked) + return (await self._session.scalars(statement)).one_or_none() + + async def database_now(self) -> datetime.datetime: + return ( + await self._session.execute(sqlmodel.select(sqlmodel.func.statement_timestamp())) + ).scalar_one() + + async def enabled_ids(self): + return tuple( + await self._session.scalars( + sqlmodel.select(CronModel.id).where(sqlmodel.col(CronModel.enabled).is_(True)) + ) + ) + + async def list(self, *, limit: int | None, cursor: int | None): + statement = sqlmodel.select(CronModel).order_by(sqlmodel.col(CronModel.id)) + if cursor is not None: + statement = statement.where(sqlmodel.col(CronModel.id) > cursor) + if limit is not None: + statement = statement.limit(limit + 1) + rows = list((await self._session.scalars(statement)).all()) + more = limit is not None and len(rows) > limit + rows = rows[:limit] + return rows, rows[-1].id if more else None + + async def save(self, cron: CronModel) -> None: + self._session.add(cron) + await self._session.flush() + await self._session.refresh(cron) + + async def delete(self, cron: CronModel) -> None: + await self._session.delete(cron) diff --git a/inkcre_core/persistence/cron/uow.py b/inkcre_core/persistence/cron/uow.py new file mode 100644 index 00000000..abfb41c3 --- /dev/null +++ b/inkcre_core/persistence/cron/uow.py @@ -0,0 +1,24 @@ +"""A Cron occurrence and its admitted Job commit together.""" + +from collections.abc import AsyncGenerator +from contextlib import asynccontextmanager +from dataclasses import dataclass + +from inkcre_core.context import current_core +from inkcre_core.persistence.job.repository import JobRepository +from inkcre_core.persistence.job.uow import JobUnitOfWork +from inkcre_core.persistence.source.repository import SourceRepository +from .repository import CronRepository + + +@dataclass(frozen=True) +class CronUnitOfWork(JobUnitOfWork): + crons: CronRepository + + +@asynccontextmanager +async def cron_uow() -> AsyncGenerator[CronUnitOfWork, None]: + async with current_core().sessions.begin() as session: + yield CronUnitOfWork( + JobRepository(session), SourceRepository(session), CronRepository(session) + ) diff --git a/inkcre_core/persistence/deployment_config/__init__.py b/inkcre_core/persistence/deployment_config/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/inkcre_core/persistence/deployment_config/repository.py b/inkcre_core/persistence/deployment_config/repository.py new file mode 100644 index 00000000..36048196 --- /dev/null +++ b/inkcre_core/persistence/deployment_config/repository.py @@ -0,0 +1,68 @@ +"""Deployment config persistence and transaction composition.""" + +from typing import Any + +import sqlmodel +from sqlalchemy.dialects.postgresql import insert +from sqlalchemy.ext.asyncio import AsyncSession + +from inkcre_core.schemas.deployment_config import DeploymentConfigModel + + +class DeploymentConfigRepository: + def __init__(self, session: AsyncSession) -> None: + self._session = session + + async def get( + self, key: str, *, for_update: bool = False + ) -> DeploymentConfigModel | None: + statement = sqlmodel.select(DeploymentConfigModel).where( + DeploymentConfigModel.key == key + ) + if for_update: + statement = statement.with_for_update() + return (await self._session.scalars(statement)).one_or_none() + + async def list( + self, *, limit: int | None, cursor: str | None + ) -> tuple[list[DeploymentConfigModel], str | None]: + statement = sqlmodel.select(DeploymentConfigModel).order_by(DeploymentConfigModel.key) + if cursor is not None: + statement = statement.where(DeploymentConfigModel.key > cursor) + if limit is not None: + statement = statement.limit(limit + 1) + rows = list((await self._session.scalars(statement)).all()) + more = limit is not None and len(rows) > limit + rows = rows[:limit] + return rows, rows[-1].key if more else None + + async def replace( + self, key: str, schema_id: str, value: dict[str, Any] + ) -> tuple[DeploymentConfigModel, bool]: + statement = insert(DeploymentConfigModel).values( + key=key, schema_id=schema_id, value=value + ) + statement = statement.on_conflict_do_nothing( + index_elements=[DeploymentConfigModel.key] + ).returning(DeploymentConfigModel) + record = (await self._session.scalars(statement)).one_or_none() + if record is not None: + return record, True + # The insert conflict, rather than a preceding read, determines creation status. + statement = ( + sqlmodel.update(DeploymentConfigModel) + .where(sqlmodel.col(DeploymentConfigModel.key) == key) + .values(schema_id=schema_id, value=value) + .returning(DeploymentConfigModel) + ) + record = (await self._session.scalars(statement)).one() + return record, False + + async def save(self, record: DeploymentConfigModel) -> None: + self._session.add(record) + await self._session.flush() + # UPDATE timestamps are managed by database triggers, not Python defaults. + await self._session.refresh(record) + + async def delete(self, record: DeploymentConfigModel) -> None: + await self._session.delete(record) diff --git a/inkcre_core/persistence/deployment_config/uow.py b/inkcre_core/persistence/deployment_config/uow.py new file mode 100644 index 00000000..3bfc3252 --- /dev/null +++ b/inkcre_core/persistence/deployment_config/uow.py @@ -0,0 +1,12 @@ +"""Deployment configuration transaction composition.""" + +from collections.abc import AsyncGenerator +from contextlib import asynccontextmanager +from inkcre_core.context import current_core +from .repository import DeploymentConfigRepository + + +@asynccontextmanager +async def configuration_transaction() -> AsyncGenerator[DeploymentConfigRepository, None]: + async with current_core().sessions.begin() as session: + yield DeploymentConfigRepository(session) diff --git a/inkcre_core/persistence/extension/__init__.py b/inkcre_core/persistence/extension/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/inkcre_core/persistence/extension/repository.py b/inkcre_core/persistence/extension/repository.py new file mode 100644 index 00000000..91c72176 --- /dev/null +++ b/inkcre_core/persistence/extension/repository.py @@ -0,0 +1,72 @@ +"""Session-bound access to canonical Extension rows and the enabled RPC.""" + +import uuid + +import sqlalchemy +import sqlmodel +from sqlalchemy.ext.asyncio import AsyncSession + +from inkcre_core.schemas.extension import ExtensionModel, ExtensionPeerParticipationModel + + +class ExtensionRepository: + def __init__(self, session: AsyncSession) -> None: + self._session = session + + async def list(self): + rows = await self._session.scalars( + sqlmodel.select(ExtensionModel).order_by(ExtensionModel.name) + ) + return tuple(rows) + + async def get(self, name: str, *, lock: bool = False): + statement = sqlmodel.select(ExtensionModel).where(ExtensionModel.name == name) + if lock: + statement = statement.with_for_update() + return (await self._session.scalars(statement)).one_or_none() + + async def try_install_lock(self, name: str) -> bool: + result = await self._session.execute( + sqlalchemy.text("SELECT pg_try_advisory_xact_lock(hashtextextended(:name, 0))"), + {"name": name}, + ) + return bool(result.scalar_one()) + + async def save(self, row: ExtensionModel) -> None: + self._session.add(row) + await self._session.flush() + + async def delete(self, row: ExtensionModel) -> None: + await self._session.delete(row) + + async def list_participations(self, peer_id: uuid.UUID): + rows = await self._session.scalars( + sqlmodel.select(ExtensionPeerParticipationModel) + .where(ExtensionPeerParticipationModel.peer_id == peer_id) + .order_by(ExtensionPeerParticipationModel.name) + ) + return tuple(rows) + + async def list_participations_for_extension(self, name: str): + rows = await self._session.scalars( + sqlmodel.select(ExtensionPeerParticipationModel).where( + ExtensionPeerParticipationModel.name == name + ) + ) + return tuple(rows) + + async def get_participation(self, name: str, peer_id: uuid.UUID, *, lock: bool = False): + statement = sqlmodel.select(ExtensionPeerParticipationModel).where( + ExtensionPeerParticipationModel.name == name, + ExtensionPeerParticipationModel.peer_id == peer_id, + ) + if lock: + statement = statement.with_for_update() + return (await self._session.scalars(statement)).one_or_none() + + async def save_participation(self, row: ExtensionPeerParticipationModel) -> None: + self._session.add(row) + await self._session.flush() + + async def delete_participation(self, row: ExtensionPeerParticipationModel) -> None: + await self._session.delete(row) diff --git a/inkcre_core/persistence/extension/uow.py b/inkcre_core/persistence/extension/uow.py new file mode 100644 index 00000000..196c6c22 --- /dev/null +++ b/inkcre_core/persistence/extension/uow.py @@ -0,0 +1,13 @@ +"""Short Extension transaction; publication and acquisition remain outside.""" + +from collections.abc import AsyncGenerator +from contextlib import asynccontextmanager + +from inkcre_core.context import current_core +from .repository import ExtensionRepository + + +@asynccontextmanager +async def extension_uow() -> AsyncGenerator[ExtensionRepository, None]: + async with current_core().sessions.begin() as session: + yield ExtensionRepository(session) diff --git a/inkcre_core/persistence/info_base/__init__.py b/inkcre_core/persistence/info_base/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/inkcre_core/persistence/info_base/repository.py b/inkcre_core/persistence/info_base/repository.py new file mode 100644 index 00000000..a2f7a302 --- /dev/null +++ b/inkcre_core/persistence/info_base/repository.py @@ -0,0 +1,501 @@ +"""Graph persistence within a caller-owned asynchronous transaction.""" + +from collections.abc import Iterable +import random +import typing +from typing import Optional as Opt +import sqlalchemy +import sqlmodel +from inkcre_core.utils.types_ import Undefined, _undefined +from inkcre_core.schemas.info_base.block import BlockID, ResolverType +from inkcre_core.schemas.info_base.relation import RelationID +from inkcre_core.schemas.info_base.storage import StorageID + +from sqlalchemy.ext.asyncio import AsyncSession + +from inkcre_core.schemas.info_base.block import BlockForm, BlockModel +from inkcre_core.schemas.info_base.relation import RelationCreateForm, RelationModel + + +class BlockRepository: + def __init__(self, session: AsyncSession) -> None: + self._session = session + + async def create_many(self, forms: Iterable[BlockForm]) -> tuple[BlockModel, ...]: + """Insert producer fields and load database identities in one flush.""" + blocks = tuple( + BlockModel.model_validate(form.model_dump(include=set(BlockForm.model_fields))) + for form in forms + ) + if blocks: + self._session.add_all(blocks) + await self._session.flush() + return blocks + + async def get_many(self, block_ids: typing.Collection[BlockID]) -> tuple[BlockModel, ...]: + """Return the existing Blocks from a bounded identity set.""" + if not block_ids: + return () + return tuple( + ( + await self._session.scalars( + sqlmodel.select(BlockModel).where( + sqlmodel.col(BlockModel.id).in_(tuple(block_ids)) + ) + ) + ).all() + ) + + async def get_by_resolvers( + self, resolvers: typing.Collection[str], *, lock: bool = False + ) -> tuple[BlockModel, ...]: + if not resolvers: + return () + statement = sqlmodel.select(BlockModel).where( + sqlmodel.col(BlockModel.resolver).in_(tuple(resolvers)) + ) + if lock: + statement = statement.order_by(sqlmodel.col(BlockModel.id)).with_for_update() + return tuple(await self._session.scalars(statement)) + + async def save_many(self, blocks: Iterable[BlockModel]) -> None: + self._session.add_all(blocks) + await self._session.flush() + + async def get_random_many(self, count: int) -> tuple[BlockModel, ...]: + """Return up to count distinct random Blocks.""" + if count <= 0: + return () + return tuple( + ( + await self._session.scalars( + sqlmodel.select(BlockModel).order_by(sqlmodel.func.random()).limit(count) + ) + ).all() + ) + + async def get_random(self) -> Opt[BlockModel]: + """Return one existing Block without transferring all identities.""" + block_id_column = sqlmodel.col(BlockModel.id) + count = ( + await self._session.scalars(sqlmodel.select(sqlmodel.func.count(block_id_column))) + ).one() + if count == 0: + return None + offset = random.SystemRandom().randrange(count) + return ( + await self._session.scalars( + sqlmodel.select(BlockModel).order_by(block_id_column).offset(offset).limit(1) + ) + ).one() + + async def get(self, block_id: BlockID, *, lock: bool = False) -> Opt[BlockModel]: + statement = sqlmodel.select(BlockModel).where(BlockModel.id == block_id) + if lock: + statement = statement.with_for_update() + return (await self._session.scalars(statement)).one_or_none() + + async def save(self, block: BlockModel) -> None: + self._session.add(block) + await self._session.flush() + + async def edit_block( + self, + block_id: BlockID, + content: Opt[str] = None, + resolver: Opt[ResolverType] = None, + storage: Opt[StorageID] | Undefined = _undefined, + ) -> BlockModel: + """编辑块""" + block = await self.get(block_id) + if block is None: + raise ValueError("Block not found") + if content is not None: + block.content = content + if resolver is not None: + block.resolver = resolver + if storage is not _undefined: + block.storage = typing.cast(StorageID | None, storage) + self._session.add(block) + await self._session.flush() + await self._session.refresh(block) + return block + + async def delete(self, block_id: BlockID) -> bool: + """Delete a block, within the surrounding transaction.""" + block = await self.get(block_id) + if block is None: + return False + await self._session.delete(block) + await self._session.flush() + return True + + async def get_recent( + self, num: int = 10, resolver: Opt[ResolverType] = None + ) -> tuple[BlockModel, ...]: + statement = ( + sqlmodel.select(BlockModel).order_by(sqlmodel.desc(BlockModel.created_at)).limit(num) + ) + if resolver: + statement = statement.where(BlockModel.resolver == resolver) + return tuple((await self._session.scalars(statement)).all()) + + async def create(self, form: BlockForm) -> BlockModel: + return (await self.create_many((form,)))[0] + + async def find_content(self, resolver: str, content: str) -> BlockModel | None: + return ( + await self._session.scalars( + sqlmodel.select(BlockModel).where( + BlockModel.resolver == resolver, + BlockModel.content == content, + ) + ) + ).one_or_none() + + async def find_json_field( + self, + resolver: str, + field: str, + value: str, + *, + block_ids: typing.Collection[BlockID] | None = None, + ) -> tuple[BlockModel, ...]: + from inkcre_core.utils.sql import find_by_json_field + + statement = sqlmodel.select(BlockModel).where( + BlockModel.resolver == resolver, + find_by_json_field(BlockModel.content, field, value), + ) + if block_ids is not None: + statement = statement.where(sqlmodel.col(BlockModel.id).in_(tuple(block_ids))) + return tuple(await self._session.scalars(statement)) + + async def find_json_field_many( + self, + resolvers: typing.Collection[str], + field: str, + values: typing.Collection[str], + ) -> tuple[BlockModel, ...]: + """Find inline JSON records in a bounded native-identity set.""" + from sqlalchemy.dialects.postgresql import JSONB + + if not resolvers or not values: + return () + return tuple( + await self._session.scalars( + sqlmodel.select(BlockModel) + .where( + sqlmodel.col(BlockModel.resolver).in_(tuple(resolvers)), + sqlalchemy.case( + ( + sqlalchemy.func.pg_input_is_valid(BlockModel.content, "jsonb"), + sqlalchemy.cast(BlockModel.content, JSONB)[field].astext, + ) + ).in_(tuple(values)), + sqlmodel.col(BlockModel.storage).is_(None), + ) + .order_by(sqlmodel.col(BlockModel.id)) + ) + ) + + async def acquire_transaction_lock(self, namespace: str, *, shared: bool = False) -> None: + """Serialize an owning use case until its surrounding transaction ends.""" + function = ( + sqlmodel.func.pg_advisory_xact_lock_shared + if shared + else sqlmodel.func.pg_advisory_xact_lock + ) + await self._session.execute( + sqlmodel.select(function(sqlmodel.func.hashtextextended(namespace, 0))) + ) + + async def get_related( + self, block_id: BlockID, *, content: str, outgoing: bool = True + ) -> BlockModel | None: + endpoint = RelationModel.from_ if outgoing else RelationModel.to_ + other = RelationModel.to_ if outgoing else RelationModel.from_ + statement = ( + sqlmodel.select(BlockModel) + .join(RelationModel, sqlmodel.col(BlockModel.id) == other) + .where( + endpoint == block_id, + RelationModel.content == content, + ) + .limit(1) + ) + return (await self._session.scalars(statement)).first() + + async def select_ids( + self, + limit: int, + *, + exclude_resolvers: typing.Collection[str], + random_order: bool = False, + ) -> tuple[BlockID, ...]: + statement = sqlmodel.select(BlockModel.id).where( + sqlmodel.col(BlockModel.resolver).not_in(tuple(exclude_resolvers)) + ) + statement = ( + statement.order_by(sqlalchemy.func.random()) + if random_order + else statement.order_by( + sqlmodel.desc(BlockModel.updated_at), sqlmodel.desc(BlockModel.id) + ) + ) + return tuple( + typing.cast(BlockID, value) + for value in await self._session.scalars(statement.limit(limit)) + ) + + async def matching_content(self, resolver: str, content: str) -> tuple[BlockModel, ...]: + return tuple( + await self._session.scalars( + sqlmodel.select(BlockModel).where( + BlockModel.resolver == resolver, BlockModel.content == content + ) + ) + ) + + async def without_outgoing_relation( + self, resolvers: typing.Collection[str], content: str, *, limit: int + ) -> tuple[BlockModel, ...]: + exists = sqlalchemy.exists( + sqlmodel.select(RelationModel.id).where( + RelationModel.from_ == BlockModel.id, RelationModel.content == content + ) + ) + return tuple( + await self._session.scalars( + sqlmodel.select(BlockModel) + .where(sqlmodel.col(BlockModel.resolver).in_(tuple(resolvers)), ~exists) + .order_by(sqlmodel.col(BlockModel.id)) + .limit(limit) + ) + ) + + +class RelationRepository: + def __init__(self, session: AsyncSession) -> None: + self._session = session + + async def create_many( + self, forms: Iterable[RelationCreateForm] + ) -> tuple[RelationModel, ...]: + """Insert directed relations without committing the surrounding operation.""" + relations = tuple(RelationModel.model_validate(form) for form in forms) + if relations: + self._session.add_all(relations) + await self._session.flush() + return relations + + async def get_many( + self, relation_ids: typing.Collection[RelationID] + ) -> tuple[RelationModel, ...]: + """Return the existing Relations from a bounded identity set.""" + if not relation_ids: + return () + return tuple( + ( + await self._session.scalars( + sqlmodel.select(RelationModel).where( + sqlmodel.col(RelationModel.id).in_(tuple(relation_ids)) + ) + ) + ).all() + ) + + async def get_for_endpoints( + self, block_ids: typing.Collection[BlockID], *, contents: typing.Collection[str] + ) -> tuple[RelationModel, ...]: + if not block_ids: + return () + return tuple( + await self._session.scalars( + sqlmodel.select(RelationModel).where( + sqlmodel.col(RelationModel.content).in_(tuple(contents)), + sqlalchemy.or_( + sqlmodel.col(RelationModel.from_).in_(tuple(block_ids)), + sqlmodel.col(RelationModel.to_).in_(tuple(block_ids)), + ), + ) + ) + ) + + async def delete_many(self, relation_ids: typing.Collection[RelationID]) -> None: + if relation_ids: + await self._session.execute( + sqlalchemy.delete(RelationModel).where( + sqlmodel.col(RelationModel.id).in_(tuple(relation_ids)) + ) + ) + + async def get_outgoing_many( + self, + block_ids: typing.Collection[BlockID], + *, + to_ids: typing.Collection[BlockID] | None = None, + content: str | None = None, + lock: bool = False, + ) -> tuple[RelationModel, ...]: + if not block_ids: + return () + statement = sqlmodel.select(RelationModel).where( + sqlmodel.col(RelationModel.from_).in_(tuple(block_ids)) + ) + if to_ids is not None: + statement = statement.where(sqlmodel.col(RelationModel.to_).in_(tuple(to_ids))) + if content is not None: + statement = statement.where(RelationModel.content == content) + if lock: + statement = statement.order_by(sqlmodel.col(RelationModel.id)).with_for_update() + return tuple(await self._session.scalars(statement)) + + async def save(self, relation: RelationModel) -> None: + self._session.add(relation) + await self._session.flush() + + async def remove(self, relation: RelationModel) -> None: + await self._session.delete(relation) + + async def get_by_id(self, relation_id: RelationID) -> RelationModel | None: + return await self._session.get(RelationModel, relation_id) + + async def get_endpoint_page( + self, + block_ids: typing.Collection[BlockID], + *, + endpoint: typing.Literal["from", "to"], + contents: typing.Collection[str] = (), + cursor: RelationID | None = None, + limit: int = 21, + ) -> tuple[RelationModel, ...]: + """Read a bounded Relation page for one persisted endpoint direction.""" + if not block_ids or limit <= 0: + return () + endpoint_column = RelationModel.from_ if endpoint == "from" else RelationModel.to_ + statement = sqlmodel.select(RelationModel).where( + sqlmodel.col(endpoint_column).in_(tuple(block_ids)) + ) + if contents: + statement = statement.where(sqlmodel.col(RelationModel.content).in_(tuple(contents))) + relation_id_column = sqlmodel.col(RelationModel.id) + if cursor is not None: + statement = statement.where(relation_id_column < cursor) + statement = statement.order_by(sqlmodel.desc(relation_id_column)).limit(limit) + return tuple((await self._session.scalars(statement)).all()) + + async def fetchsert(self, relation: RelationModel) -> RelationModel: + """Create if not exists, else return the existing one. + + Will NOT commit the session. + """ + existing = ( + await self._session.scalars( + sqlmodel.select(RelationModel).where( + RelationModel.content == relation.content, + RelationModel.from_ == relation.from_, + RelationModel.to_ == relation.to_, + ) + ) + ).one_or_none() + if existing is not None: + return existing + self._session.add(relation) + await self._session.flush() + return relation + + async def get( + self, + block_id: BlockID, + include_in: bool = True, + include_out: bool = True, + content: Opt[str] = None, + ) -> tuple[RelationModel, ...]: + """Get relations from/to a block + + :param include_in: Include the relations where the block is the target + :param include_out: Include the relations where the block is the source + :param content: If specified, filter relations by content (eq) + """ + if not include_in and (not include_out): + return () + directions: list[typing.Any] = [] + if include_in: + directions.append(RelationModel.to_ == block_id) + if include_out: + directions.append(RelationModel.from_ == block_id) + statement = sqlmodel.select(RelationModel).where(sqlalchemy.or_(*directions)) + if content is not None: + statement = statement.where(RelationModel.content == content) + res = (await self._session.scalars(statement)).all() + return tuple(res) + + async def update( + self, + relation_id: RelationID, + *, + from_: BlockID | Undefined = _undefined, + to_: BlockID | Undefined = _undefined, + content: str | Undefined = _undefined, + ) -> RelationModel: + """Update selected relation facts in the surrounding transaction.""" + relation = ( + await self._session.scalars( + sqlmodel.select(RelationModel).where(RelationModel.id == relation_id) + ) + ).one_or_none() + if relation is None: + raise ValueError("Relation not found") + if from_ is not _undefined: + relation.from_ = typing.cast(BlockID, from_) + if to_ is not _undefined: + relation.to_ = typing.cast(BlockID, to_) + if content is not _undefined: + relation.content = typing.cast(str, content) + self._session.add(relation) + await self._session.flush() + await self._session.refresh(relation) + return relation + + async def delete(self, relation_id: RelationID) -> bool: + """Delete a relation, within the surrounding transaction.""" + relation = ( + await self._session.scalars( + sqlmodel.select(RelationModel).where(RelationModel.id == relation_id) + ) + ).one_or_none() + if relation is None: + return False + await self._session.delete(relation) + await self._session.flush() + return True + + async def create(self, from_: BlockID, to_: BlockID, content: str) -> RelationModel: + return ( + await self.create_many((RelationCreateForm(from_=from_, to_=to_, content=content),)) + )[0] + + async def random_incoming_sources( + self, block_id: BlockID, content: str, limit: int + ) -> tuple[BlockID, ...]: + return tuple( + await self._session.scalars( + sqlmodel.select(RelationModel.from_) + .where(RelationModel.to_ == block_id, RelationModel.content == content) + .order_by(sqlalchemy.func.random()) + .limit(limit) + ) + ) + + async def recent( + self, limit: int, *, contents: typing.Collection[str] = () + ) -> tuple[RelationModel, ...]: + statement = ( + sqlmodel.select(RelationModel) + .order_by(sqlmodel.desc(RelationModel.updated_at), sqlmodel.desc(RelationModel.id)) + .limit(limit) + ) + if contents: + statement = statement.where(sqlmodel.col(RelationModel.content).in_(tuple(contents))) + return tuple(await self._session.scalars(statement)) diff --git a/inkcre_core/persistence/info_base/storage.py b/inkcre_core/persistence/info_base/storage.py new file mode 100644 index 00000000..33067ab0 --- /dev/null +++ b/inkcre_core/persistence/info_base/storage.py @@ -0,0 +1,96 @@ +"""Storage catalog and byte persistence bound to the enclosing graph transaction.""" + +from collections.abc import Sequence +from typing import Any +from uuid import UUID + +import sqlalchemy +from sqlalchemy.dialects.postgresql import insert +from sqlalchemy.ext.asyncio import AsyncSession +import sqlmodel + +from inkcre_core.schemas.info_base.storage import ( + StorageBlobModel, + StorageModel, + StorageTypesModel, +) + + +class StorageRepository: + def __init__(self, session: AsyncSession) -> None: + self._session = session + + async def get(self, storage_id: int) -> StorageModel: + return ( + await self._session.scalars( + sqlmodel.select(StorageModel).where(StorageModel.id == storage_id) + ) + ).one() + + async def sync_types(self, values: Sequence[dict[str, Any]]) -> None: + if not values: + return + statement = insert(StorageTypesModel).values(values) + await self._session.execute( + statement.on_conflict_do_update( + index_elements=[StorageTypesModel.id], + set_={ + "description": statement.excluded.description, + "config_schema": statement.excluded.config_schema, + "writable": statement.excluded.writable, + }, + ) + ) + + async def sync_builtins(self, values: Sequence[dict[str, Any]]) -> None: + if not values: + return + statement = insert(StorageModel).values(values) + await self._session.execute( + statement.on_conflict_do_update( + index_elements=["id"], + set_={ + "type": statement.excluded.type, + "nickname": statement.excluded.nickname, + "config": statement.excluded.config, + }, + ) + ) + + async def create(self, storage: StorageModel) -> StorageModel: + self._session.add(storage) + await self._session.flush() + return storage + + async def read_blob(self, blob_id: UUID, *, lock: bool = False) -> bytes | None: + statement = sqlmodel.select(StorageBlobModel.data).where(StorageBlobModel.id == blob_id) + if lock: + statement = statement.with_for_update() + return (await self._session.scalars(statement)).one_or_none() + + async def create_blob(self, content: bytes) -> UUID: + blob = StorageBlobModel(data=content) + self._session.add(blob) + await self._session.flush() + return blob.id + + async def update_blob(self, blob_id: UUID, content: bytes) -> bool: + result = await self._session.scalars( + sqlalchemy.update(StorageBlobModel) + .where( + sqlmodel.col(StorageBlobModel.id) == blob_id, + ) + .values(data=content) + .returning(sqlmodel.col(StorageBlobModel.id)) + ) + return result.one_or_none() is not None + + async def delete_blob(self, blob_id: UUID) -> bool: + result = await self._session.scalars( + sqlalchemy.delete(StorageBlobModel) + .where( + sqlmodel.col(StorageBlobModel.id) == blob_id, + ) + .returning(sqlmodel.col(StorageBlobModel.id)) + ) + return result.one_or_none() is not None diff --git a/inkcre_core/persistence/info_base/uow.py b/inkcre_core/persistence/info_base/uow.py new file mode 100644 index 00000000..ae4f95eb --- /dev/null +++ b/inkcre_core/persistence/info_base/uow.py @@ -0,0 +1,30 @@ +"""Graph transaction composition; repositories share one private session.""" + +from collections.abc import AsyncGenerator +from contextlib import asynccontextmanager +from dataclasses import dataclass + +from inkcre_core.context import current_core + +from .repository import BlockRepository, RelationRepository +from .storage import StorageRepository + + +@dataclass(frozen=True) +class GraphUnitOfWork: + blocks: BlockRepository + relations: RelationRepository + storage: StorageRepository + + +@asynccontextmanager +async def graph_uow( + *, repeatable_read: bool = False +) -> AsyncGenerator[GraphUnitOfWork, None]: + """Commit on success; roll back on failure and always close the session.""" + async with current_core().sessions.begin() as session: + if repeatable_read: + await session.connection(execution_options={"isolation_level": "REPEATABLE READ"}) + yield GraphUnitOfWork( + BlockRepository(session), RelationRepository(session), StorageRepository(session) + ) diff --git a/inkcre_core/persistence/job/__init__.py b/inkcre_core/persistence/job/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/inkcre_core/persistence/job/repository.py b/inkcre_core/persistence/job/repository.py new file mode 100644 index 00000000..803dc841 --- /dev/null +++ b/inkcre_core/persistence/job/repository.py @@ -0,0 +1,140 @@ +"""Session-bound durable Job catalog, claim and terminal writes.""" + +import typing + +import sqlalchemy +from sqlalchemy.dialects.postgresql import insert +from sqlalchemy.ext.asyncio import AsyncSession +import sqlmodel + +from inkcre_core.schemas.job import JobModel, JobStatus, JobTypeModel + + +class JobRepository: + def __init__(self, session: AsyncSession) -> None: + self._session = session + + async def sync_types(self, rows: list[dict]) -> None: + if not rows: + return + statement = insert(JobTypeModel).values(rows) + await self._session.execute( + statement.on_conflict_do_update( + index_elements=[JobTypeModel.id], + set_={ + key: getattr(statement.excluded, key) + for key in ("description", "parameters_schema", "default_timeout_seconds") + }, + ) + ) + + async def get(self, job_id: int) -> JobModel | None: + return await self._session.get(JobModel, job_id) + + async def get_type(self, type_: str) -> JobTypeModel | None: + return await self._session.get(JobTypeModel, type_) + + async def create(self, job: JobModel) -> JobModel: + self._session.add(job) + await self._session.flush() + await self._session.refresh(job) + return job + + async def list_types(self, *, limit: int | None, cursor: str | None): + statement = sqlmodel.select(JobTypeModel).order_by(JobTypeModel.id) + if cursor is not None: + statement = statement.where(JobTypeModel.id > cursor) + if limit is not None: + statement = statement.limit(limit + 1) + rows = list((await self._session.scalars(statement)).all()) + more = limit is not None and len(rows) > limit + rows = rows[:limit] + return rows, rows[-1].id if more else None + + async def list_jobs( + self, *, limit: int, cursor: int | None, type_: str | None, status: JobStatus | None + ): + statement = sqlmodel.select(JobModel).order_by(sqlmodel.col(JobModel.id).desc()) + if cursor is not None: + statement = statement.where(sqlmodel.col(JobModel.id) < cursor) + if type_ is not None: + statement = statement.where(JobModel.type == type_) + if status is not None: + statement = statement.where(JobModel.status == status) + rows = list((await self._session.scalars(statement.limit(limit + 1))).all()) + more = len(rows) > limit + rows = rows[:limit] + return rows, rows[-1].id if more else None + + async def abort(self, job_id: int) -> JobModel | None: + table = typing.cast(typing.Any, getattr(JobModel, "__table__")) + await self._session.execute( + sqlalchemy.update(table) + .where( + table.c.id == job_id, + table.c.status.in_((JobStatus.PENDING, JobStatus.RUNNING)), + ) + .values( + abort_requested=True, + status=sqlalchemy.case( + ( + table.c.status == JobStatus.PENDING, + sqlalchemy.cast(JobStatus.ABORTED.value, table.c.status.type), + ), + else_=table.c.status, + ), + ) + ) + return await self.get(job_id) + + async def abort_requests(self, active_ids: tuple[int, ...]): + return tuple( + await self._session.scalars( + sqlmodel.select(JobModel.id).where( + sqlmodel.col(JobModel.id).in_(active_ids), + sqlmodel.col(JobModel.abort_requested).is_(True), + ) + ) + ) + + async def claim(self, job_id: int) -> JobModel | None: + result = await self._session.scalars( + sqlalchemy.update(JobModel) + .where( + sqlmodel.col(JobModel.id) == job_id, + sqlmodel.col(JobModel.status) == JobStatus.PENDING, + ) + .values(status=JobStatus.RUNNING) + .returning(JobModel) + ) + return result.one_or_none() + + async def close(self, job: JobModel, status: JobStatus) -> bool: + table = typing.cast(typing.Any, getattr(JobModel, "__table__")) + result = await self._session.execute( + sqlalchemy.update(table) + .where(table.c.id == job.id, table.c.status == JobStatus.RUNNING) + .values(status=status, state=job.state) + ) + return bool(typing.cast(typing.Any, result).rowcount) + + async def expire_overdue(self) -> int: + table = typing.cast(typing.Any, getattr(JobModel, "__table__")) + result = await self._session.execute( + sqlalchemy.update(table) + .where( + table.c.status == JobStatus.RUNNING, + sqlalchemy.text( + "started_at + timeout_seconds * interval '1 second' <= statement_timestamp()" + ), + ) + .values(status=JobStatus.TIMED_OUT) + ) + return typing.cast(typing.Any, result).rowcount or 0 + + async def pending(self): + return tuple( + await self._session.scalars( + sqlmodel.select(JobModel).where(JobModel.status == JobStatus.PENDING) + ) + ) diff --git a/inkcre_core/persistence/job/uow.py b/inkcre_core/persistence/job/uow.py new file mode 100644 index 00000000..75530fad --- /dev/null +++ b/inkcre_core/persistence/job/uow.py @@ -0,0 +1,21 @@ +"""Job admission also reads Source contracts within its transaction.""" + +from collections.abc import AsyncGenerator +from contextlib import asynccontextmanager +from dataclasses import dataclass + +from inkcre_core.context import current_core +from inkcre_core.persistence.source.repository import SourceRepository +from .repository import JobRepository + + +@dataclass(frozen=True) +class JobUnitOfWork: + jobs: JobRepository + sources: SourceRepository + + +@asynccontextmanager +async def job_uow() -> AsyncGenerator[JobUnitOfWork, None]: + async with current_core().sessions.begin() as session: + yield JobUnitOfWork(JobRepository(session), SourceRepository(session)) diff --git a/inkcre_core/persistence/lexical_retrieval/__init__.py b/inkcre_core/persistence/lexical_retrieval/__init__.py new file mode 100644 index 00000000..05dca4e7 --- /dev/null +++ b/inkcre_core/persistence/lexical_retrieval/__init__.py @@ -0,0 +1 @@ +"""Lexical projection persistence.""" diff --git a/inkcre_core/persistence/lexical_retrieval/repository.py b/inkcre_core/persistence/lexical_retrieval/repository.py new file mode 100644 index 00000000..1322b007 --- /dev/null +++ b/inkcre_core/persistence/lexical_retrieval/repository.py @@ -0,0 +1,145 @@ +"""Session-bound lexical ranking, candidate scans and projection batch writes.""" + +import datetime +import typing + +import sqlalchemy +from sqlalchemy.dialects.postgresql import insert +from sqlalchemy.ext.asyncio import AsyncSession +import sqlmodel + +from inkcre_core.schemas.info_base.block import BlockModel +from inkcre_core.schemas.lexical_retrieval import BlockLexicalRecordModel + + +class LexicalRepository: + def __init__(self, session: AsyncSession) -> None: + self._session = session + + async def search(self, normalized: str, limit: int): + literal_pattern = f"%{self._escape_like(normalized)}%" + + block_columns = typing.cast( + typing.Any, + BlockModel.__table__.c, # pyrefly: ignore[missing-attribute] + ) + record_columns = typing.cast( + typing.Any, + BlockLexicalRecordModel.__table__.c, # pyrefly: ignore[missing-attribute] + ) + query_terms = sqlalchemy.func.plainto_tsquery("simple", normalized) + label_exact = sqlalchemy.func.lower(record_columns.label) == normalized.lower() + label_substring = record_columns.label.ilike(literal_pattern, escape="\\") + text_substring = record_columns.text.ilike(literal_pattern, escape="\\") + term_match = record_columns.search_vector.op("@@")(query_terms) + evidence = sqlalchemy.case( + (label_exact, "label_exact"), + (label_substring, "label_substring"), + (text_substring, "text_substring"), + else_="terms", + ).label("evidence") + evidence_class = sqlalchemy.case( + (label_exact, 4.0), + (label_substring, 3.0), + (text_substring, 2.0), + else_=1.0, + ) + term_rank = sqlalchemy.func.ts_rank_cd(record_columns.search_vector, query_terms) + rank = (evidence_class + term_rank).label("rank") + + statement = ( + sqlmodel.select(BlockModel, BlockLexicalRecordModel, evidence, rank) + .join( + BlockLexicalRecordModel, + record_columns.block == block_columns.id, + ) + .where( + record_columns.updated_at >= block_columns.updated_at, + sqlalchemy.or_( + label_exact, + label_substring, + text_substring, + term_match, + ), + ) + .order_by( + sqlalchemy.desc(evidence_class), + sqlalchemy.desc(term_rank), + block_columns.id, + ) + .limit(limit) + ) + return (await self._session.execute(statement)).all() + + async def candidate_page( + self, + cursor: int, + page_size: int, + rebuild_cutoff: datetime.datetime | None, + ) -> tuple[tuple[BlockModel, ...], int, bool]: + block_columns = typing.cast( + typing.Any, + BlockModel.__table__.c, # pyrefly: ignore[missing-attribute] + ) + record_columns = typing.cast( + typing.Any, + BlockLexicalRecordModel.__table__.c, # pyrefly: ignore[missing-attribute] + ) + needs_projection = sqlalchemy.or_( + record_columns.block.is_(None), + record_columns.updated_at < block_columns.updated_at, + ) + if rebuild_cutoff is not None: + needs_projection = sqlalchemy.or_( + needs_projection, + record_columns.updated_at < rebuild_cutoff, + ) + statement = ( + sqlmodel.select(BlockModel) + .outerjoin( + BlockLexicalRecordModel, + record_columns.block == block_columns.id, + ) + .where(block_columns.id > cursor, needs_projection) + .order_by(block_columns.id) + .limit(page_size) + ) + blocks = tuple(await self._session.scalars(statement)) + next_cursor = typing.cast(int, blocks[-1].id) if blocks else cursor + return blocks, next_cursor, len(blocks) < page_size + + async def database_now(self) -> datetime.datetime: + return ( + await self._session.execute(sqlmodel.select(sqlalchemy.func.current_timestamp())) + ).scalar_one() + + async def upsert(self, projections: list[dict[str, typing.Any]]) -> None: + if not projections: + return + rows = [] + for projection in projections: + label_vector = sqlalchemy.func.setweight( + sqlalchemy.func.to_tsvector("simple", projection["label"]), + sqlalchemy.literal_column("'A'::\"char\""), + ) + text_vector = sqlalchemy.func.setweight( + sqlalchemy.func.to_tsvector("simple", projection["text"] or ""), + sqlalchemy.literal_column("'D'::\"char\""), + ) + rows.append({**projection, "search_vector": label_vector.op("||")(text_vector)}) + statement = insert(BlockLexicalRecordModel).values(rows) + await self._session.execute( + statement.on_conflict_do_update( + index_elements=["block"], + set_={ + "label": statement.excluded.label, + "text": statement.excluded.text, + "search_vector": statement.excluded.search_vector, + "updated_at": sqlalchemy.func.current_timestamp(), + }, + ) + ) + + @staticmethod + def _escape_like(value: str) -> str: + return value.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") diff --git a/inkcre_core/persistence/lexical_retrieval/uow.py b/inkcre_core/persistence/lexical_retrieval/uow.py new file mode 100644 index 00000000..818a22c8 --- /dev/null +++ b/inkcre_core/persistence/lexical_retrieval/uow.py @@ -0,0 +1,13 @@ +"""Short lexical database scope, separate from Resolver projection.""" + +from collections.abc import AsyncGenerator +from contextlib import asynccontextmanager + +from inkcre_core.context import current_core +from .repository import LexicalRepository + + +@asynccontextmanager +async def lexical_uow() -> AsyncGenerator[LexicalRepository, None]: + async with current_core().sessions.begin() as session: + yield LexicalRepository(session) diff --git a/inkcre_core/persistence/observability/__init__.py b/inkcre_core/persistence/observability/__init__.py new file mode 100644 index 00000000..302d1365 --- /dev/null +++ b/inkcre_core/persistence/observability/__init__.py @@ -0,0 +1 @@ +"""Isolated asynchronous telemetry persistence.""" diff --git a/inkcre_core/persistence/observability/uow.py b/inkcre_core/persistence/observability/uow.py new file mode 100644 index 00000000..e442cde0 --- /dev/null +++ b/inkcre_core/persistence/observability/uow.py @@ -0,0 +1,11 @@ +"""Log batches commit independently of application transactions.""" + +from collections.abc import Sequence + +from inkcre_core.context import current_core +from inkcre_core.libs.obsrv.log_record import LogModel + + +async def write_logs(entries: Sequence[LogModel]) -> None: + async with current_core().sessions.begin() as session: + session.add_all(entries) diff --git a/inkcre_core/persistence/peer/__init__.py b/inkcre_core/persistence/peer/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/inkcre_core/persistence/peer/repository.py b/inkcre_core/persistence/peer/repository.py new file mode 100644 index 00000000..0b9b0daf --- /dev/null +++ b/inkcre_core/persistence/peer/repository.py @@ -0,0 +1,106 @@ +"""Peer persistence; all lease comparisons use the database clock.""" + +import datetime +from typing import Any + +import sqlalchemy +from sqlalchemy.dialects.postgresql import insert +from sqlalchemy.ext.asyncio import AsyncSession +import sqlmodel + +from inkcre_core.database_contract import PROTOCOL_SCHEMA +from inkcre_core.schemas.peer import PeerModel, PeerRef + + +class PeerRepository: + def __init__(self, session: AsyncSession) -> None: + self._session = session + + async def register( + self, + peer_id: PeerRef, + name: str, + application_version: str, + schema: dict, + ) -> PeerModel: + statement = insert(PeerModel).values( + id=peer_id, + name=name, + application_version=application_version, + labels=[], + config={}, + config_schema=schema, + capabilities=[], + ) + statement = statement.on_conflict_do_update( + index_elements=["id"], + set_={ + "application_version": statement.excluded.application_version, + "config_schema": statement.excluded.config_schema, + }, + ).returning(PeerModel) + return (await self._session.scalars(statement)).one() + + async def get(self, peer_id: PeerRef, *, for_update: bool = False) -> PeerModel | None: + statement = sqlmodel.select(PeerModel).where(PeerModel.id == peer_id) + if for_update: + statement = statement.with_for_update() + return (await self._session.scalars(statement)).one_or_none() + + async def get_all(self) -> tuple[PeerModel, ...]: + return tuple((await self._session.scalars(sqlmodel.select(PeerModel))).all()) + + async def with_leases( + self, + *, + peer_id: PeerRef | None = None, + limit: int | None = None, + cursor: PeerRef | None = None, + ) -> tuple[list[dict[str, Any]], PeerRef | None]: + statement = sqlmodel.select( + PeerModel, + sqlalchemy.func.coalesce( + PeerModel.lease_expires_at > sqlalchemy.func.statement_timestamp(), False + ), + ).order_by(sqlmodel.col(PeerModel.id)) + if peer_id is not None: + statement = statement.where(PeerModel.id == peer_id) + if cursor is not None: + statement = statement.where(PeerModel.id > cursor) + if limit is not None: + statement = statement.limit(limit + 1) + rows = list((await self._session.execute(statement)).all()) + more = limit is not None and len(rows) > limit + rows = rows[:limit] + return [ + {**row.model_dump(mode="json"), "lease_active": active} for row, active in rows + ], rows[-1][0].id if more else None + + async def save(self, peer: PeerModel) -> None: + self._session.add(peer) + await self._session.flush() + await self._session.refresh(peer) + + async def renew_lease(self, peer_id: PeerRef, ttl_seconds: int) -> datetime.datetime: + statement = sqlalchemy.select( + getattr(sqlalchemy.func, PROTOCOL_SCHEMA).renew_peer_lease(peer_id, ttl_seconds) + ) + return (await self._session.execute(statement)).scalar_one() + + async def clear_lease(self, peer_id: PeerRef) -> None: + await self._session.execute( + sqlalchemy.update(PeerModel) + .where(sqlmodel.col(PeerModel.id) == peer_id) + .values(lease_expires_at=None) + ) + + async def candidates( + self, self_id: PeerRef, route_to_peer: PeerRef | None + ) -> tuple[PeerModel, ...]: + statement = sqlmodel.select(PeerModel).where( + PeerModel.id != self_id, + PeerModel.lease_expires_at > sqlalchemy.func.statement_timestamp(), + ) + if route_to_peer is not None: + statement = statement.where(PeerModel.id == route_to_peer) + return tuple((await self._session.scalars(statement)).all()) diff --git a/inkcre_core/persistence/peer/uow.py b/inkcre_core/persistence/peer/uow.py new file mode 100644 index 00000000..a2b4df11 --- /dev/null +++ b/inkcre_core/persistence/peer/uow.py @@ -0,0 +1,13 @@ +"""Transaction scopes for Peer facts and database-owned leases.""" + +from collections.abc import AsyncGenerator +from contextlib import asynccontextmanager + +from inkcre_core.context import current_core +from .repository import PeerRepository + + +@asynccontextmanager +async def peer_uow() -> AsyncGenerator[PeerRepository, None]: + async with current_core().sessions.begin() as session: + yield PeerRepository(session) diff --git a/inkcre_core/persistence/semantic_retrieval/__init__.py b/inkcre_core/persistence/semantic_retrieval/__init__.py new file mode 100644 index 00000000..d396cf8b --- /dev/null +++ b/inkcre_core/persistence/semantic_retrieval/__init__.py @@ -0,0 +1 @@ +"""Semantic retrieval persistence.""" diff --git a/inkcre_core/persistence/semantic_retrieval/repository.py b/inkcre_core/persistence/semantic_retrieval/repository.py new file mode 100644 index 00000000..c27c8513 --- /dev/null +++ b/inkcre_core/persistence/semantic_retrieval/repository.py @@ -0,0 +1,203 @@ +"""Session-bound embedding profiles and derived retrieval records.""" + +import datetime +import typing + +import sqlalchemy +import sqlalchemy.orm +from sqlalchemy.dialects.postgresql import insert +from sqlalchemy.ext.asyncio import AsyncSession +import sqlmodel + +from inkcre_core.schemas.info_base.block import BlockModel +from inkcre_core.schemas.info_base.relation import RelationModel +from inkcre_core.schemas.semantic_retrieval import VectorRetrievalOptions + +from inkcre_core.schemas.ai import ( + EmbeddingProfileID, + EmbeddingProfileModel, + BlockEmbeddingModel, + RelationEmbeddingModel, +) + + +class SemanticRetrievalRepository: + def __init__(self, session: AsyncSession) -> None: + self._session = session + + async def get_profile( + self, profile_id: EmbeddingProfileID + ) -> EmbeddingProfileModel | None: + return await self._session.get(EmbeddingProfileModel, profile_id) + + async def list_profiles(self, *, limit: int | None, cursor: int | None): + statement = sqlmodel.select(EmbeddingProfileModel).order_by( + sqlmodel.col(EmbeddingProfileModel.id) + ) + if cursor is not None: + statement = statement.where(sqlmodel.col(EmbeddingProfileModel.id) > cursor) + if limit is not None: + statement = statement.limit(limit + 1) + rows = list(await self._session.scalars(statement)) + more = limit is not None and len(rows) > limit + rows = rows[:limit] + return rows, rows[-1].id if more else None + + async def database_now(self) -> datetime.datetime: + return ( + await self._session.execute(sqlmodel.select(sqlalchemy.func.current_timestamp())) + ).scalar_one() + + async def candidate_blocks(self, profile_id: int, cursor: int, page_size: int): + block_columns = typing.cast( + typing.Any, + BlockModel.__table__.c, # pyrefly: ignore[missing-attribute] + ) + record_columns = typing.cast( + typing.Any, + BlockEmbeddingModel.__table__.c, # pyrefly: ignore[missing-attribute] + ) + statement = ( + sqlmodel.select(BlockModel, BlockEmbeddingModel) + .outerjoin( + BlockEmbeddingModel, + sqlalchemy.and_( + record_columns.profile == profile_id, + record_columns.block == block_columns.id, + ), + ) + .where(block_columns.id > cursor) + .order_by(block_columns.id) + .limit(page_size) + ) + return (await self._session.execute(statement)).all() + + async def candidate_relations(self, profile_id: int, cursor: int, page_size: int): + from_block = sqlalchemy.orm.aliased(BlockModel) + to_block = sqlalchemy.orm.aliased(BlockModel) + from_columns = typing.cast(typing.Any, from_block) + to_columns = typing.cast(typing.Any, to_block) + relation_columns = typing.cast( + typing.Any, + RelationModel.__table__.c, # pyrefly: ignore[missing-attribute] + ) + record_columns = typing.cast( + typing.Any, + RelationEmbeddingModel.__table__.c, # pyrefly: ignore[missing-attribute] + ) + statement = ( + sqlmodel.select( + RelationModel, + RelationEmbeddingModel, + from_block.updated_at, + to_block.updated_at, + ) + .outerjoin( + RelationEmbeddingModel, + sqlalchemy.and_( + record_columns.profile == profile_id, + record_columns.relation == relation_columns.id, + ), + ) + .join(from_block, from_columns.id == relation_columns.from_) + .join(to_block, to_columns.id == relation_columns.to_) + .where(relation_columns.id > cursor) + .order_by(relation_columns.id) + .limit(page_size) + ) + return (await self._session.execute(statement)).all() + + async def retrieve_blocks( + self, + profile: EmbeddingProfileModel, + query_vector, + options: VectorRetrievalOptions, + ): + profile_id = profile.id + block_columns = typing.cast( + typing.Any, + BlockModel.__table__.c, # pyrefly: ignore[missing-attribute] + ) + record_columns = typing.cast( + typing.Any, + BlockEmbeddingModel.__table__.c, # pyrefly: ignore[missing-attribute] + ) + distance = record_columns.embedding.cosine_distance(query_vector).label("distance") + statement = ( + sqlmodel.select(BlockModel, distance) + .join(BlockEmbeddingModel, record_columns.block == block_columns.id) + .where( + record_columns.profile == profile_id, + record_columns.updated_at >= profile.updated_at, + record_columns.updated_at >= block_columns.updated_at, + sqlalchemy.func.vector_dims(record_columns.embedding) == profile.dimensions, + sqlalchemy.func.vector_norm(record_columns.embedding) > 0, + ) + .order_by(distance, block_columns.id) + .limit(options.limit) + ) + if options.min_score is not None: + statement = statement.where(distance <= 1 - options.min_score) + return (await self._session.execute(statement)).all() + + async def retrieve_relations( + self, + profile: EmbeddingProfileModel, + query_vector, + options: VectorRetrievalOptions, + ): + profile_id = profile.id + from_block = sqlalchemy.orm.aliased(BlockModel) + to_block = sqlalchemy.orm.aliased(BlockModel) + from_columns = typing.cast(typing.Any, from_block) + to_columns = typing.cast(typing.Any, to_block) + relation_columns = typing.cast( + typing.Any, + RelationModel.__table__.c, # pyrefly: ignore[missing-attribute] + ) + record_columns = typing.cast( + typing.Any, + RelationEmbeddingModel.__table__.c, # pyrefly: ignore[missing-attribute] + ) + distance = record_columns.embedding.cosine_distance(query_vector).label("distance") + statement = ( + sqlmodel.select(RelationModel, distance) + .join( + RelationEmbeddingModel, + record_columns.relation == relation_columns.id, + ) + .join(from_block, from_columns.id == relation_columns.from_) + .join(to_block, to_columns.id == relation_columns.to_) + .where( + record_columns.profile == profile_id, + record_columns.updated_at >= profile.updated_at, + record_columns.updated_at >= relation_columns.updated_at, + record_columns.updated_at >= from_columns.updated_at, + record_columns.updated_at >= to_columns.updated_at, + sqlalchemy.func.vector_dims(record_columns.embedding) == profile.dimensions, + sqlalchemy.func.vector_norm(record_columns.embedding) > 0, + ) + .order_by(distance, relation_columns.id) + .limit(options.limit) + ) + if options.min_score is not None: + statement = statement.where(distance <= 1 - options.min_score) + return (await self._session.execute(statement)).all() + + async def upsert(self, blocks: list[dict], relations: list[dict]) -> None: + for model, key, rows in ( + (BlockEmbeddingModel, "block", blocks), + (RelationEmbeddingModel, "relation", relations), + ): + if not rows: + continue + statement = insert(model).values(rows) + await self._session.execute( + statement.on_conflict_do_update( + index_elements=["profile", key], + set_={ + "embedding": statement.excluded.embedding, + "updated_at": sqlalchemy.func.current_timestamp(), + }, + ) + ) diff --git a/inkcre_core/persistence/semantic_retrieval/uow.py b/inkcre_core/persistence/semantic_retrieval/uow.py new file mode 100644 index 00000000..41a0a587 --- /dev/null +++ b/inkcre_core/persistence/semantic_retrieval/uow.py @@ -0,0 +1,13 @@ +"""Short persistence scope; provider calls run after it closes.""" + +from collections.abc import AsyncGenerator +from contextlib import asynccontextmanager + +from inkcre_core.context import current_core +from .repository import SemanticRetrievalRepository + + +@asynccontextmanager +async def semantic_retrieval_uow() -> AsyncGenerator[SemanticRetrievalRepository, None]: + async with current_core().sessions.begin() as session: + yield SemanticRetrievalRepository(session) diff --git a/inkcre_core/persistence/sink/__init__.py b/inkcre_core/persistence/sink/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/inkcre_core/persistence/sink/repository.py b/inkcre_core/persistence/sink/repository.py new file mode 100644 index 00000000..c90304c1 --- /dev/null +++ b/inkcre_core/persistence/sink/repository.py @@ -0,0 +1,55 @@ +"""Session-bound Sink catalog and instance records.""" + +import sqlmodel +from sqlalchemy.dialects.postgresql import insert +from sqlalchemy.ext.asyncio import AsyncSession + +from inkcre_core.schemas.sink import SinkModel, SinkTypeModel + + +class SinkRepository: + def __init__(self, session: AsyncSession) -> None: + self._session = session + + async def sync_types(self, rows: list[dict]) -> None: + if not rows: + return + statement = insert(SinkTypeModel).values(rows) + await self._session.execute( + statement.on_conflict_do_update( + index_elements=["id"], + set_={ + "description": statement.excluded.description, + "config_schema": statement.excluded.config_schema, + }, + ) + ) + + async def list_types(self): + return tuple( + await self._session.scalars(sqlmodel.select(SinkTypeModel).order_by(SinkTypeModel.id)) + ) + + async def get_type(self, sink_type: str) -> SinkTypeModel | None: + return await self._session.get(SinkTypeModel, sink_type) + + async def list(self): + return tuple( + await self._session.scalars( + sqlmodel.select(SinkModel).order_by(sqlmodel.col(SinkModel.id)) + ) + ) + + async def get(self, sink_id: int, *, lock: bool = False) -> SinkModel | None: + statement = sqlmodel.select(SinkModel).where(SinkModel.id == sink_id) + if lock: + statement = statement.with_for_update() + return (await self._session.scalars(statement)).one_or_none() + + async def save(self, sink: SinkModel) -> None: + self._session.add(sink) + await self._session.flush() + await self._session.refresh(sink) + + async def delete(self, sink: SinkModel) -> None: + await self._session.delete(sink) diff --git a/inkcre_core/persistence/sink/uow.py b/inkcre_core/persistence/sink/uow.py new file mode 100644 index 00000000..0cf49554 --- /dev/null +++ b/inkcre_core/persistence/sink/uow.py @@ -0,0 +1,13 @@ +"""Short Sink persistence scope; runtime start/close happen after commit.""" + +from collections.abc import AsyncGenerator +from contextlib import asynccontextmanager + +from inkcre_core.context import current_core +from .repository import SinkRepository + + +@asynccontextmanager +async def sink_uow() -> AsyncGenerator[SinkRepository, None]: + async with current_core().sessions.begin() as session: + yield SinkRepository(session) diff --git a/inkcre_core/persistence/source/__init__.py b/inkcre_core/persistence/source/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/inkcre_core/persistence/source/repository.py b/inkcre_core/persistence/source/repository.py new file mode 100644 index 00000000..eb767123 --- /dev/null +++ b/inkcre_core/persistence/source/repository.py @@ -0,0 +1,76 @@ +"""Persistence for Source catalog and instances.""" + +import sqlmodel +from sqlalchemy.dialects.postgresql import insert +from sqlalchemy.ext.asyncio import AsyncSession + +from inkcre_core.schemas.source import SourceModel, SourceTypesModel + + +class SourceRepository: + def __init__(self, session: AsyncSession) -> None: + self._session = session + + async def sync_types(self, rows: list[dict]) -> None: + if not rows: + return + statement = insert(SourceTypesModel).values(rows) + statement = statement.on_conflict_do_update( + index_elements=[SourceTypesModel.id], + set_={ + key: getattr(statement.excluded, key) + for key in ( + "description", + "config_schema", + "collect_config_schema", + "backfill_config_schema", + ) + }, + ) + await self._session.execute(statement) + + async def get(self, source_id: int, *, lock: bool = False): + statement = sqlmodel.select(SourceModel).where(SourceModel.id == source_id) + if lock: + statement = statement.with_for_update() + return (await self._session.scalars(statement)).one_or_none() + + async def get_by_block(self, block_id: int) -> SourceModel | None: + return ( + await self._session.scalars( + sqlmodel.select(SourceModel).where(SourceModel.block == block_id) + ) + ).one_or_none() + + async def save(self, source: SourceModel) -> None: + self._session.add(source) + await self._session.flush() + await self._session.refresh(source) + + async def get_type(self, type_: str) -> SourceTypesModel | None: + return await self._session.get(SourceTypesModel, type_) + + async def list_sources(self, *, limit: int | None, cursor: int | None): + statement = sqlmodel.select(SourceModel).order_by(sqlmodel.col(SourceModel.id)) + if cursor is not None: + statement = statement.where(sqlmodel.col(SourceModel.id) > cursor) + if limit is not None: + statement = statement.limit(limit + 1) + rows = list((await self._session.scalars(statement)).all()) + more = limit is not None and len(rows) > limit + rows = rows[:limit] + return rows, rows[-1].id if more else None + + async def list_types(self, *, limit: int | None, cursor: str | None): + statement = sqlmodel.select(SourceTypesModel).order_by(SourceTypesModel.id) + if cursor is not None: + statement = statement.where(SourceTypesModel.id > cursor) + if limit is not None: + statement = statement.limit(limit + 1) + rows = list((await self._session.scalars(statement)).all()) + more = limit is not None and len(rows) > limit + rows = rows[:limit] + return rows, rows[-1].id if more else None + + async def delete(self, source: SourceModel) -> None: + await self._session.delete(source) diff --git a/inkcre_core/persistence/source/uow.py b/inkcre_core/persistence/source/uow.py new file mode 100644 index 00000000..d2215124 --- /dev/null +++ b/inkcre_core/persistence/source/uow.py @@ -0,0 +1,31 @@ +"""Source state, graph and storage participate in one short transaction.""" + +from collections.abc import AsyncGenerator +from contextlib import asynccontextmanager +from dataclasses import dataclass + +from inkcre_core.context import current_core +from inkcre_core.persistence.info_base.uow import GraphUnitOfWork +from inkcre_core.persistence.info_base.repository import BlockRepository, RelationRepository +from inkcre_core.persistence.info_base.storage import StorageRepository +from inkcre_core.persistence.deployment_config.repository import DeploymentConfigRepository +from .repository import SourceRepository + + +@dataclass(frozen=True) +class SourceUnitOfWork: + sources: SourceRepository + graph: GraphUnitOfWork + configuration: DeploymentConfigRepository + + +@asynccontextmanager +async def source_uow() -> AsyncGenerator[SourceUnitOfWork, None]: + async with current_core().sessions.begin() as session: + yield SourceUnitOfWork( + SourceRepository(session), + GraphUnitOfWork( + BlockRepository(session), RelationRepository(session), StorageRepository(session) + ), + DeploymentConfigRepository(session), + ) diff --git a/inkcre_core/py.typed b/inkcre_core/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/app/schemas/AGENTS.md b/inkcre_core/schemas/AGENTS.md similarity index 76% rename from app/schemas/AGENTS.md rename to inkcre_core/schemas/AGENTS.md index 658aa47f..effb9057 100644 --- a/app/schemas/AGENTS.md +++ b/inkcre_core/schemas/AGENTS.md @@ -1,4 +1,4 @@ -- Import your schema in `app/schemas/__init__.py` so Alembic discovers it while generating migrations. +- Import your schema in `inkcre_core/schemas/__init__.py` so Alembic discovers it while generating migrations. - Follow the [Python input/restoration boundary](../../docs/30-unit-tdd/business-pipeline-and-authority.md#cross-subtree-constraints) when changing Pydantic forms or persistence codecs. A typed input should not be dumped and revalidated merely to cross an internal call boundary; necessary restoration of nested persisted types is a different operation. diff --git a/inkcre_core/schemas/__init__.py b/inkcre_core/schemas/__init__.py new file mode 100644 index 00000000..c7eb0f62 --- /dev/null +++ b/inkcre_core/schemas/__init__.py @@ -0,0 +1,56 @@ +__all__ = [ + "Base", + "BlockModel", + "StorageModel", + "StorageBlobModel", + "StorageTypesModel", + "RelationModel", + "SourceModel", + "JobTypeModel", + "JobModel", + "CronModel", + "ExtensionModel", + "ExtensionPeerParticipationModel", + "AIDialectModel", + "AIProviderModel", + "AIModelModel", + "EmbeddingProfileModel", + "RelationEmbeddingModel", + "BlockEmbeddingModel", + "BlockLexicalRecordModel", + "AgentDefinitionModel", + "PeerModel", + "DeploymentConfigModel", + "SinkModel", + "SinkTypeModel", +] + +import sqlalchemy.orm +import sqlmodel + +from inkcre_core.database_contract import PROTOCOL_SCHEMA + + +Base = sqlalchemy.orm.declarative_base(metadata=sqlalchemy.MetaData(schema=PROTOCOL_SCHEMA)) +sqlmodel.SQLModel.metadata = Base.metadata + +from .info_base.block import BlockModel +from .info_base.storage import StorageBlobModel, StorageModel, StorageTypesModel +from .info_base.relation import RelationModel +from .source import SourceModel +from .job import JobModel, JobTypeModel +from .cron import CronModel +from .extension.main import ExtensionModel, ExtensionPeerParticipationModel +from .ai import ( + AIDialectModel, + AIModelModel, + AIProviderModel, + BlockEmbeddingModel, + EmbeddingProfileModel, + RelationEmbeddingModel, +) +from .agent import AgentDefinitionModel +from .peer import PeerModel +from .deployment_config import DeploymentConfigModel +from .lexical_retrieval import BlockLexicalRecordModel +from .sink import SinkModel, SinkTypeModel diff --git a/inkcre_core/schemas/agent.py b/inkcre_core/schemas/agent.py new file mode 100644 index 00000000..aaefd3b8 --- /dev/null +++ b/inkcre_core/schemas/agent.py @@ -0,0 +1,173 @@ +"""Reusable Agent definitions persisted as shared deployment facts.""" + +import datetime +import typing + +import pydantic +import sqlalchemy +import sqlalchemy.dialects.postgresql +import sqlmodel + +from inkcre_core.schemas.ai import AIModelID, ToolChoice + + +AgentID: typing.TypeAlias = int + +_TOOL_CHOICE_ADAPTER = pydantic.TypeAdapter(ToolChoice | None) + + +def normalize_agent_tools(value: typing.Iterable[str]) -> tuple[str, ...]: + """Validate and canonicalize Agent Tool IDs with set semantics.""" + tools = tuple(value) + if any(not tool for tool in tools): + raise ValueError("Agent Tool IDs must not be empty") + if len(tools) != len(set(tools)): + raise ValueError("Agent Tool IDs must be unique") + return tuple(sorted(tools)) + + +class AgentToolsType(sqlalchemy.TypeDecorator): + """Round-trip canonical Agent Tool ID sets through PostgreSQL text arrays.""" + + impl = sqlalchemy.dialects.postgresql.ARRAY(sqlalchemy.Text) + cache_ok = True + + def process_bind_param(self, value, dialect): + del dialect + return list(normalize_agent_tools(value or ())) + + def process_result_value(self, value, dialect): + del dialect + return tuple(value or ()) + + +class ToolChoiceType(sqlalchemy.TypeDecorator): + """Round-trip the canonical nullable ToolChoice union through JSONB.""" + + impl = sqlalchemy.dialects.postgresql.JSONB + cache_ok = True + + def load_dialect_impl(self, dialect): + return dialect.type_descriptor(sqlalchemy.dialects.postgresql.JSONB(none_as_null=True)) + + def process_bind_param(self, value, dialect): + del dialect + choice = _TOOL_CHOICE_ADAPTER.validate_python(value) + if isinstance(choice, pydantic.BaseModel): + return choice.model_dump(mode="json") + return choice + + def process_result_value(self, value, dialect): + del dialect + return _TOOL_CHOICE_ADAPTER.validate_python(value) + + +class AgentForm(pydantic.BaseModel): + """Editable Agent definition, without execution or database-managed fields.""" + + model_config = pydantic.ConfigDict(extra="forbid") + + name: str + system_prompt: str + model: AIModelID + tools: tuple[str, ...] = () + tool_choice: ToolChoice | None = None + max_model_calls_per_turn: int = pydantic.Field(gt=0) + + @pydantic.field_validator("tools") + @classmethod + def canonical_tools(cls, value: tuple[str, ...]) -> tuple[str, ...]: + return normalize_agent_tools(value) + + +class AgentUpdateForm(pydantic.BaseModel): + """PATCH omission preserves a field; only tool_choice accepts explicit null.""" + + model_config = pydantic.ConfigDict(extra="forbid") + + name: str = pydantic.Field(default=None) # pyrefly: ignore[bad-assignment] + system_prompt: str = pydantic.Field(default=None) # pyrefly: ignore[bad-assignment] + model: AIModelID = pydantic.Field(default=None) # pyrefly: ignore[bad-assignment] + tools: tuple[str, ...] = () + tool_choice: ToolChoice | None = None + max_model_calls_per_turn: int = pydantic.Field(default=None, gt=0) # pyrefly: ignore[bad-assignment] + + +class AgentDefinitionModel(sqlmodel.SQLModel, table=True): + """One reusable system-prompt, model, Tool and turn-budget composition.""" + + __tablename__ = "agents" # type: ignore + __table_args__ = ( + sqlalchemy.CheckConstraint( + "max_model_calls_per_turn > 0", + name="ck_agents_max_model_calls_per_turn_positive", + ), + ) + + id: AgentID | None = sqlmodel.Field( + default=None, + sa_column=sqlalchemy.Column( + sqlalchemy.BigInteger, + sqlalchemy.Identity(), + primary_key=True, + ), + ) + name: str = sqlmodel.Field(sa_column=sqlalchemy.Column(sqlalchemy.Text, nullable=False)) + system_prompt: str = sqlmodel.Field( + sa_column=sqlalchemy.Column(sqlalchemy.Text, nullable=False) + ) + tools: tuple[str, ...] = sqlmodel.Field( + default=(), + sa_column=sqlalchemy.Column( + AgentToolsType(), + nullable=False, + server_default=sqlalchemy.text("'{}'::text[]"), + ), + ) + tool_choice: ToolChoice | None = sqlmodel.Field( + default=None, + sa_column=sqlalchemy.Column(ToolChoiceType(), nullable=True), + ) + model: AIModelID = sqlmodel.Field( + sa_column=sqlalchemy.Column( + sqlalchemy.BigInteger, + sqlalchemy.ForeignKey("ai_models.id", onupdate="CASCADE", ondelete="RESTRICT"), + nullable=False, + ) + ) + max_model_calls_per_turn: int = sqlmodel.Field( + sa_column=sqlalchemy.Column(sqlalchemy.Integer, nullable=False) + ) + created_at: datetime.datetime = sqlmodel.Field( + default=None, + sa_column=sqlalchemy.Column( + sqlalchemy.TIMESTAMP(timezone=True), + nullable=False, + server_default=sqlalchemy.text("CURRENT_TIMESTAMP"), + ), + ) + updated_at: datetime.datetime = sqlmodel.Field( + default=None, + sa_column=sqlalchemy.Column( + sqlalchemy.TIMESTAMP(timezone=True), + nullable=False, + server_default=sqlalchemy.text("CURRENT_TIMESTAMP"), + ), + ) + + @pydantic.field_validator("tools", mode="before") + @classmethod + def canonical_tools(cls, value: typing.Iterable[str]) -> tuple[str, ...]: + return normalize_agent_tools(value) + + @pydantic.field_validator("tool_choice", mode="before") + @classmethod + def canonical_tool_choice(cls, value: typing.Any) -> ToolChoice | None: + return _TOOL_CHOICE_ADAPTER.validate_python(value) + + @pydantic.field_validator("max_model_calls_per_turn") + @classmethod + def positive_model_call_budget(cls, value: int) -> int: + if value <= 0: + raise ValueError("max_model_calls_per_turn must be positive") + return value diff --git a/inkcre_core/schemas/ai/__init__.py b/inkcre_core/schemas/ai/__init__.py new file mode 100644 index 00000000..e226e18e --- /dev/null +++ b/inkcre_core/schemas/ai/__init__.py @@ -0,0 +1,80 @@ +"""Canonical AI contracts and shared persistence models.""" + +from .capability import ( + AICapability, + AICapabilityType, + AIFeature, + AIModality, + AIModelCapability, + ChatCapability, + EmbeddingCapability, + normalize_capabilities, +) +from .chat import ( + AudioContentPart, + AssistantMessage, + FunctionTool, + ImageContentPart, + JSONValue, + Message, + NamedToolChoice, + SystemMessage, + TextContentPart, + ToolCall, + ToolChoice, + ToolResult, + ToolResultMessage, + UserContentPart, + UserMessage, + VideoContentPart, + validate_message_history, +) +from .embedding import BlockEmbeddingModel, RelationEmbeddingModel +from .main import ( + AIDialectID, + AIDialectModel, + AIModelID, + AIModelModel, + AIProviderID, + AIProviderModel, + EmbeddingProfileID, + EmbeddingProfileModel, +) + +__all__ = [ + "AICapability", + "AICapabilityType", + "AIDialectID", + "AIDialectModel", + "AIFeature", + "AIModality", + "AIModelCapability", + "AIModelID", + "AIModelModel", + "AIProviderID", + "AIProviderModel", + "AudioContentPart", + "AssistantMessage", + "BlockEmbeddingModel", + "ChatCapability", + "EmbeddingCapability", + "EmbeddingProfileID", + "EmbeddingProfileModel", + "FunctionTool", + "ImageContentPart", + "JSONValue", + "Message", + "NamedToolChoice", + "RelationEmbeddingModel", + "SystemMessage", + "TextContentPart", + "ToolCall", + "ToolChoice", + "ToolResult", + "ToolResultMessage", + "UserContentPart", + "UserMessage", + "VideoContentPart", + "normalize_capabilities", + "validate_message_history", +] diff --git a/inkcre_core/schemas/ai/capability.py b/inkcre_core/schemas/ai/capability.py new file mode 100644 index 00000000..4a52afcf --- /dev/null +++ b/inkcre_core/schemas/ai/capability.py @@ -0,0 +1,67 @@ +"""Provider-neutral AI model capability declarations.""" + +import typing + +import pydantic + + +AICapabilityType: typing.TypeAlias = typing.Literal["embedding", "chat"] +AIModality: typing.TypeAlias = str +AIFeature: typing.TypeAlias = str + + +def _canonical_string_set(value: typing.Any) -> tuple[str, ...]: + if not isinstance(value, (list, tuple, set, frozenset)): + raise ValueError("value must be an array of strings") + items = tuple(value) + if not all(isinstance(item, str) and item for item in items): + raise ValueError("values must be non-empty strings") + if len(items) != len(set(items)): + raise ValueError("duplicate values are not allowed") + return tuple(sorted(items)) + + +class AICapability(pydantic.BaseModel): + """Fields shared by every typed capability declaration.""" + + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + type: AICapabilityType + input_modalities: tuple[AIModality, ...] + output_modalities: tuple[AIModality, ...] + features: tuple[AIFeature, ...] = () + + _normalize_input_modalities = pydantic.field_validator("input_modalities", mode="before")( + _canonical_string_set + ) + _normalize_output_modalities = pydantic.field_validator( + "output_modalities", mode="before" + )(_canonical_string_set) + _normalize_features = pydantic.field_validator("features", mode="before")( + _canonical_string_set + ) + + +class EmbeddingCapability(AICapability): + type: typing.Literal["embedding"] = "embedding" + + +class ChatCapability(AICapability): + type: typing.Literal["chat"] = "chat" + + +AIModelCapability: typing.TypeAlias = typing.Annotated[ + EmbeddingCapability | ChatCapability, + pydantic.Field(discriminator="type"), +] + +CAPABILITIES_ADAPTER = pydantic.TypeAdapter(tuple[AIModelCapability, ...]) + + +def normalize_capabilities(value: typing.Any) -> tuple[AIModelCapability, ...]: + """Validate and canonically order one declaration per capability type.""" + capabilities = CAPABILITIES_ADAPTER.validate_python(value) + types = tuple(capability.type for capability in capabilities) + if len(types) != len(set(types)): + raise ValueError("duplicate AI capability types are not allowed") + return tuple(sorted(capabilities, key=lambda capability: capability.type)) diff --git a/inkcre_core/schemas/ai/chat.py b/inkcre_core/schemas/ai/chat.py new file mode 100644 index 00000000..013ef9ad --- /dev/null +++ b/inkcre_core/schemas/ai/chat.py @@ -0,0 +1,214 @@ +"""Canonical AI chat, Tool and ToolResult contracts.""" + +import typing +from urllib.parse import urlsplit + +import pydantic + + +JSONValue: typing.TypeAlias = pydantic.JsonValue + + +class FunctionTool(pydantic.BaseModel): + """One caller-owned function Tool exposed to an AI model.""" + + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + id: str = pydantic.Field(min_length=1) + description: str + input_schema: dict[str, JSONValue] + + +class ToolCall(pydantic.BaseModel): + """One model-authored request to invoke a caller-owned Tool.""" + + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + id: str = pydantic.Field(min_length=1) + tool: str = pydantic.Field(min_length=1) + arguments: dict[str, JSONValue] + + +class ToolResult(pydantic.BaseModel): + """One caller-side result correlated to a ToolCall.""" + + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + tool_call_id: str = pydantic.Field(min_length=1) + content: JSONValue + is_error: bool = False + + +class NamedToolChoice(pydantic.BaseModel): + """Require one exact caller-owned Tool.""" + + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + type: typing.Literal["function"] = "function" + tool: str = pydantic.Field(min_length=1) + + +ToolChoice: typing.TypeAlias = typing.Literal["none", "auto", "required"] | NamedToolChoice + + +class SystemMessage(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + type: typing.Literal["system"] = "system" + content: str + + +class TextContentPart(pydantic.BaseModel): + """One authored text item in an ordered multimodal User message.""" + + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + type: typing.Literal["text"] = "text" + text: str = pydantic.Field(min_length=1) + + +class _MediaContentPart(pydantic.BaseModel): + """AI-owned media transfer value, independent of Block and Storage identity.""" + + model_config = pydantic.ConfigDict( + extra="forbid", + frozen=True, + ser_json_bytes="base64", + val_json_bytes="base64", + ) + + data: bytes = pydantic.Field(min_length=1) + mime_type: str = pydantic.Field(min_length=1) + transfer_url: str | None = None + + @pydantic.field_validator("mime_type") + @classmethod + def normalized_mime_type(cls, value: str) -> str: + normalized = value.partition(";")[0].strip().lower() + if "/" not in normalized or any(character.isspace() for character in normalized): + raise ValueError("mime_type must be one concrete media type") + return normalized + + @pydantic.field_validator("transfer_url") + @classmethod + def http_transfer_url(cls, value: str | None) -> str | None: + if value is None: + return None + parsed = urlsplit(value) + if parsed.scheme not in {"http", "https"} or not parsed.netloc: + raise ValueError("transfer_url must be one absolute HTTP(S) URL") + if parsed.username is not None or parsed.password is not None: + raise ValueError("transfer_url must not contain user-info credentials") + return value + + +class ImageContentPart(_MediaContentPart): + type: typing.Literal["image"] = "image" + + @pydantic.field_validator("mime_type") + @classmethod + def image_mime_type(cls, value: str) -> str: + if not value.startswith("image/"): + raise ValueError("ImageContentPart requires an image MIME type") + return value + + +class AudioContentPart(_MediaContentPart): + type: typing.Literal["audio"] = "audio" + + @pydantic.field_validator("mime_type") + @classmethod + def audio_mime_type(cls, value: str) -> str: + if not value.startswith("audio/"): + raise ValueError("AudioContentPart requires an audio MIME type") + return value + + +class VideoContentPart(_MediaContentPart): + type: typing.Literal["video"] = "video" + + @pydantic.field_validator("mime_type") + @classmethod + def video_mime_type(cls, value: str) -> str: + if not value.startswith("video/"): + raise ValueError("VideoContentPart requires a video MIME type") + return value + + +UserContentPart: typing.TypeAlias = typing.Annotated[ + TextContentPart | ImageContentPart | AudioContentPart | VideoContentPart, + pydantic.Field(discriminator="type"), +] + + +class UserMessage(pydantic.BaseModel): + model_config = pydantic.ConfigDict( + extra="forbid", + frozen=True, + ser_json_bytes="base64", + val_json_bytes="base64", + ) + + type: typing.Literal["user"] = "user" + content: tuple[UserContentPart, ...] = pydantic.Field(min_length=1) + + +class AssistantMessage(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + type: typing.Literal["assistant"] = "assistant" + content: str | None = None + tool_calls: tuple[ToolCall, ...] = () + + @pydantic.model_validator(mode="after") + def unique_tool_call_ids(self) -> "AssistantMessage": + call_ids = tuple(call.id for call in self.tool_calls) + if len(call_ids) != len(set(call_ids)): + raise ValueError("AssistantMessage ToolCall IDs must be unique") + return self + + +class ToolResultMessage(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + type: typing.Literal["tool_result"] = "tool_result" + results: tuple[ToolResult, ...] = pydantic.Field(min_length=1) + + @pydantic.model_validator(mode="after") + def unique_tool_call_ids(self) -> "ToolResultMessage": + call_ids = tuple(result.tool_call_id for result in self.results) + if len(call_ids) != len(set(call_ids)): + raise ValueError("ToolResultMessage ToolCall IDs must be unique") + return self + + +Message: typing.TypeAlias = typing.Annotated[ + SystemMessage | UserMessage | AssistantMessage | ToolResultMessage, + pydantic.Field(discriminator="type"), +] + +_MESSAGES_ADAPTER = pydantic.TypeAdapter(tuple[Message, ...]) + + +def validate_message_history(value: typing.Any) -> tuple[Message, ...]: + """Validate canonical types and every closed Assistant/ToolResult pair.""" + messages = _MESSAGES_ADAPTER.validate_python(value) + index = 0 + while index < len(messages): + message = messages[index] + if isinstance(message, ToolResultMessage): + raise ValueError("ToolResultMessage must follow an AssistantMessage with ToolCalls") + if isinstance(message, AssistantMessage) and message.tool_calls: + if index + 1 >= len(messages) or not isinstance( + messages[index + 1], ToolResultMessage + ): + raise ValueError("AssistantMessage ToolCalls require an adjacent ToolResultMessage") + result_message = typing.cast(ToolResultMessage, messages[index + 1]) + call_ids = {call.id for call in message.tool_calls} + result_ids = {result.tool_call_id for result in result_message.results} + if result_ids != call_ids: + raise ValueError("ToolResultMessage must exactly cover preceding ToolCall IDs") + index += 2 + continue + index += 1 + return messages diff --git a/inkcre_core/schemas/ai/embedding.py b/inkcre_core/schemas/ai/embedding.py new file mode 100644 index 00000000..72e2aafc --- /dev/null +++ b/inkcre_core/schemas/ai/embedding.py @@ -0,0 +1,91 @@ +"""Profile-scoped Block and Relation embedding records.""" + +import datetime + +import pgvector.sqlalchemy +import sqlalchemy +import sqlmodel + +from inkcre_core.schemas.info_base.main import Vector + +from .main import EmbeddingProfileID + + +class BlockEmbeddingModel(sqlmodel.SQLModel, table=True): + __tablename__ = "block_embeddings" # type: ignore + + profile: EmbeddingProfileID = sqlmodel.Field( + sa_column=sqlalchemy.Column( + sqlalchemy.BigInteger, + sqlalchemy.ForeignKey( + "embedding_profiles.id", onupdate="CASCADE", ondelete="RESTRICT" + ), + primary_key=True, + ) + ) + block: int = sqlmodel.Field( + sa_column=sqlalchemy.Column( + sqlalchemy.Integer, + sqlalchemy.ForeignKey("blocks.id", onupdate="CASCADE", ondelete="CASCADE"), + primary_key=True, + ) + ) + embedding: Vector = sqlmodel.Field( + sa_column=sqlalchemy.Column(pgvector.sqlalchemy.VECTOR(), nullable=False) + ) + created_at: datetime.datetime = sqlmodel.Field( + default=None, + sa_column=sqlalchemy.Column( + sqlalchemy.TIMESTAMP(timezone=True), + nullable=False, + server_default=sqlalchemy.text("CURRENT_TIMESTAMP"), + ), + ) + updated_at: datetime.datetime = sqlmodel.Field( + default=None, + sa_column=sqlalchemy.Column( + sqlalchemy.TIMESTAMP(timezone=True), + nullable=False, + server_default=sqlalchemy.text("CURRENT_TIMESTAMP"), + ), + ) + + +class RelationEmbeddingModel(sqlmodel.SQLModel, table=True): + __tablename__ = "relation_embeddings" # type: ignore + + profile: EmbeddingProfileID = sqlmodel.Field( + sa_column=sqlalchemy.Column( + sqlalchemy.BigInteger, + sqlalchemy.ForeignKey( + "embedding_profiles.id", onupdate="CASCADE", ondelete="RESTRICT" + ), + primary_key=True, + ) + ) + relation: int = sqlmodel.Field( + sa_column=sqlalchemy.Column( + sqlalchemy.Integer, + sqlalchemy.ForeignKey("relations.id", onupdate="CASCADE", ondelete="CASCADE"), + primary_key=True, + ) + ) + embedding: Vector = sqlmodel.Field( + sa_column=sqlalchemy.Column(pgvector.sqlalchemy.VECTOR(), nullable=False) + ) + created_at: datetime.datetime = sqlmodel.Field( + default=None, + sa_column=sqlalchemy.Column( + sqlalchemy.TIMESTAMP(timezone=True), + nullable=False, + server_default=sqlalchemy.text("CURRENT_TIMESTAMP"), + ), + ) + updated_at: datetime.datetime = sqlmodel.Field( + default=None, + sa_column=sqlalchemy.Column( + sqlalchemy.TIMESTAMP(timezone=True), + nullable=False, + server_default=sqlalchemy.text("CURRENT_TIMESTAMP"), + ), + ) diff --git a/inkcre_core/schemas/ai/main.py b/inkcre_core/schemas/ai/main.py new file mode 100644 index 00000000..f189d7eb --- /dev/null +++ b/inkcre_core/schemas/ai/main.py @@ -0,0 +1,227 @@ +"""Shared AI dialect, provider, model and embedding-profile facts.""" + +import datetime +import typing + +import pydantic +import sqlalchemy +import sqlalchemy.dialects.postgresql +import sqlmodel + +from .capability import CAPABILITIES_ADAPTER, AIModelCapability, normalize_capabilities + + +AIDialectID: typing.TypeAlias = str +AIProviderID: typing.TypeAlias = int +AIModelID: typing.TypeAlias = int +EmbeddingProfileID: typing.TypeAlias = int + + +class AICapabilitiesType(sqlalchemy.TypeDecorator): + """Round-trip typed, canonical capability declarations through JSONB.""" + + impl = sqlalchemy.dialects.postgresql.JSONB + cache_ok = True + + def process_bind_param(self, value, dialect): + del dialect + return [ + capability.model_dump(mode="json") for capability in normalize_capabilities(value) + ] + + def process_result_value(self, value, dialect): + del dialect + return CAPABILITIES_ADAPTER.validate_python(value or ()) + + +class AIDialectModel(sqlmodel.SQLModel, table=True): + """One exact shared AI wire/config dialect contract.""" + + __tablename__ = "ai_dialects" # type: ignore + + id: AIDialectID = sqlmodel.Field( + sa_column=sqlalchemy.Column(sqlalchemy.Text, primary_key=True) + ) + description: str = sqlmodel.Field( + sa_column=sqlalchemy.Column(sqlalchemy.Text, nullable=False) + ) + config_schema: dict[str, typing.Any] = sqlmodel.Field( + default_factory=dict, + sa_column=sqlalchemy.Column( + sqlalchemy.dialects.postgresql.JSONB, + nullable=False, + server_default=sqlalchemy.text("'{}'::jsonb"), + ), + ) + + +class AIProviderModel(sqlmodel.SQLModel, table=True): + """One deployment-scoped configuration of an AI dialect.""" + + __tablename__ = "ai_providers" # type: ignore + + id: AIProviderID | None = sqlmodel.Field( + default=None, + sa_column=sqlalchemy.Column( + sqlalchemy.BigInteger, + sqlalchemy.Identity(), + primary_key=True, + ), + ) + name: str = sqlmodel.Field(sa_column=sqlalchemy.Column(sqlalchemy.Text, nullable=False)) + dialect: AIDialectID = sqlmodel.Field( + sa_column=sqlalchemy.Column( + sqlalchemy.Text, + sqlalchemy.ForeignKey("ai_dialects.id", onupdate="CASCADE", ondelete="RESTRICT"), + nullable=False, + ) + ) + config: dict[str, typing.Any] = sqlmodel.Field( + default_factory=dict, + sa_column=sqlalchemy.Column( + sqlalchemy.dialects.postgresql.JSONB, + nullable=False, + server_default=sqlalchemy.text("'{}'::jsonb"), + ), + ) + enabled: bool = sqlmodel.Field( + default=True, + sa_column=sqlalchemy.Column( + sqlalchemy.Boolean, + nullable=False, + server_default=sqlalchemy.true(), + ), + ) + created_at: datetime.datetime = sqlmodel.Field( + default=None, + sa_column=sqlalchemy.Column( + sqlalchemy.TIMESTAMP(timezone=True), + nullable=False, + server_default=sqlalchemy.text("CURRENT_TIMESTAMP"), + ), + ) + updated_at: datetime.datetime = sqlmodel.Field( + default=None, + sa_column=sqlalchemy.Column( + sqlalchemy.TIMESTAMP(timezone=True), + nullable=False, + server_default=sqlalchemy.text("CURRENT_TIMESTAMP"), + ), + ) + + +class AIModelModel(sqlmodel.SQLModel, table=True): + """One provider-bound native model offering and its effective capabilities.""" + + __tablename__ = "ai_models" # type: ignore + __table_args__ = ( + sqlalchemy.UniqueConstraint( + "provider", "native_model_id", name="uq_ai_models_provider_native_model" + ), + ) + + id: AIModelID | None = sqlmodel.Field( + default=None, + sa_column=sqlalchemy.Column( + sqlalchemy.BigInteger, + sqlalchemy.Identity(), + primary_key=True, + ), + ) + provider: AIProviderID = sqlmodel.Field( + sa_column=sqlalchemy.Column( + sqlalchemy.BigInteger, + sqlalchemy.ForeignKey("ai_providers.id", onupdate="CASCADE", ondelete="RESTRICT"), + nullable=False, + ) + ) + native_model_id: str = sqlmodel.Field( + sa_column=sqlalchemy.Column(sqlalchemy.Text, nullable=False) + ) + name: str | None = sqlmodel.Field( + default=None, + sa_column=sqlalchemy.Column(sqlalchemy.Text, nullable=True), + ) + capabilities: tuple[AIModelCapability, ...] = sqlmodel.Field( + default=(), + sa_column=sqlalchemy.Column(AICapabilitiesType(), nullable=False), + ) + enabled: bool = sqlmodel.Field( + default=True, + sa_column=sqlalchemy.Column( + sqlalchemy.Boolean, + nullable=False, + server_default=sqlalchemy.true(), + ), + ) + created_at: datetime.datetime = sqlmodel.Field( + default=None, + sa_column=sqlalchemy.Column( + sqlalchemy.TIMESTAMP(timezone=True), + nullable=False, + server_default=sqlalchemy.text("CURRENT_TIMESTAMP"), + ), + ) + updated_at: datetime.datetime = sqlmodel.Field( + default=None, + sa_column=sqlalchemy.Column( + sqlalchemy.TIMESTAMP(timezone=True), + nullable=False, + server_default=sqlalchemy.text("CURRENT_TIMESTAMP"), + ), + ) + + @pydantic.field_validator("capabilities", mode="before") + @classmethod + def canonical_capabilities(cls, value: typing.Any) -> tuple[AIModelCapability, ...]: + return normalize_capabilities(value) + + +class EmbeddingProfileModel(sqlmodel.SQLModel, table=True): + """One durable vector-space compatibility contract.""" + + __tablename__ = "embedding_profiles" # type: ignore + __table_args__ = ( + sqlalchemy.CheckConstraint( + "dimensions > 0", name="ck_embedding_profiles_dimensions_positive" + ), + ) + + id: EmbeddingProfileID | None = sqlmodel.Field( + default=None, + sa_column=sqlalchemy.Column( + sqlalchemy.BigInteger, + sqlalchemy.Identity(), + primary_key=True, + ), + ) + name: str | None = sqlmodel.Field( + default=None, + sa_column=sqlalchemy.Column(sqlalchemy.Text, nullable=True), + ) + ai_model: AIModelID = sqlmodel.Field( + sa_column=sqlalchemy.Column( + sqlalchemy.BigInteger, + sqlalchemy.ForeignKey("ai_models.id", onupdate="CASCADE", ondelete="RESTRICT"), + nullable=False, + ) + ) + dimensions: int = sqlmodel.Field( + sa_column=sqlalchemy.Column(sqlalchemy.Integer, nullable=False) + ) + created_at: datetime.datetime = sqlmodel.Field( + default=None, + sa_column=sqlalchemy.Column( + sqlalchemy.TIMESTAMP(timezone=True), + nullable=False, + server_default=sqlalchemy.text("CURRENT_TIMESTAMP"), + ), + ) + updated_at: datetime.datetime = sqlmodel.Field( + default=None, + sa_column=sqlalchemy.Column( + sqlalchemy.TIMESTAMP(timezone=True), + nullable=False, + server_default=sqlalchemy.text("CURRENT_TIMESTAMP"), + ), + ) diff --git a/inkcre_core/schemas/cron.py b/inkcre_core/schemas/cron.py new file mode 100644 index 00000000..bf452e15 --- /dev/null +++ b/inkcre_core/schemas/cron.py @@ -0,0 +1,120 @@ +"""Deployment-wide recurring Job-template schemas.""" + +import datetime +import typing + +import pydantic +import sqlalchemy +import sqlalchemy.dialects.postgresql +import sqlmodel + +from .job import JobID, JobTypeID + + +CronID: typing.TypeAlias = int + + +class CronModel(sqlmodel.SQLModel, table=True): + """One five-field UNIX schedule that materializes typed Jobs.""" + + __tablename__ = "crons" # type: ignore + __table_args__ = ( + sqlalchemy.Index("crons_enabled_idx", "enabled"), + sqlalchemy.CheckConstraint( + "job_timeout_seconds IS NULL OR job_timeout_seconds > 0", + name="crons_job_timeout_seconds_positive", + ), + ) + + id: CronID | None = sqlmodel.Field( + default=None, + sa_column=sqlalchemy.Column( + sqlalchemy.BigInteger, + sqlalchemy.Identity(), + primary_key=True, + ), + ) + schedule: str = sqlmodel.Field( + min_length=1, + sa_column=sqlalchemy.Column(sqlalchemy.Text, nullable=False), + ) + enabled: bool = sqlmodel.Field( + default=True, + sa_column=sqlalchemy.Column( + sqlalchemy.Boolean, + nullable=False, + server_default=sqlalchemy.true(), + ), + ) + job_type: JobTypeID = sqlmodel.Field( + sa_column=sqlalchemy.Column( + sqlalchemy.Text, + sqlalchemy.ForeignKey("job_types.id", onupdate="CASCADE", ondelete="RESTRICT"), + nullable=False, + ) + ) + job_parameters: dict[str, typing.Any] = sqlmodel.Field( + default_factory=dict, + sa_column=sqlalchemy.Column( + sqlalchemy.dialects.postgresql.JSONB, + nullable=False, + server_default=sqlalchemy.text("'{}'::jsonb"), + ), + ) + job_timeout_seconds: int | None = sqlmodel.Field( + default=None, + gt=0, + sa_column=sqlalchemy.Column(sqlalchemy.Integer, nullable=True), + ) + last_job: JobID | None = sqlmodel.Field( + default=None, + sa_column=sqlalchemy.Column( + sqlalchemy.BigInteger, + sqlalchemy.ForeignKey("jobs.id", onupdate="CASCADE", ondelete="RESTRICT"), + nullable=True, + ), + ) + last_scheduled_for: datetime.datetime | None = sqlmodel.Field( + default=None, + sa_column=sqlalchemy.Column(sqlalchemy.TIMESTAMP(timezone=True), nullable=True), + ) + created_at: datetime.datetime = sqlmodel.Field( + default=None, + sa_column=sqlalchemy.Column( + sqlalchemy.TIMESTAMP(timezone=True), + nullable=False, + server_default=sqlalchemy.text("CURRENT_TIMESTAMP"), + ), + ) + updated_at: datetime.datetime = sqlmodel.Field( + default=None, + sa_column=sqlalchemy.Column( + sqlalchemy.TIMESTAMP(timezone=True), + nullable=False, + server_default=sqlalchemy.text("CURRENT_TIMESTAMP"), + ), + ) + + +class CronForm(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid") + + schedule: str = pydantic.Field(min_length=1) + enabled: bool = True + job_type: JobTypeID + job_parameters: dict[str, typing.Any] = pydantic.Field(default_factory=dict) + job_timeout_seconds: int | None = pydantic.Field(default=None, gt=0) + + +class CronUpdateForm(pydantic.BaseModel): + """Optional submitted fields; null only clears the timeout override.""" + + model_config = pydantic.ConfigDict(extra="forbid") + + # Omission is not null input: exclude_unset removes these defaults, whereas + # an explicitly submitted null still fails the non-null field type. + schedule: str = pydantic.Field(default=None, min_length=1) # pyrefly: ignore[bad-assignment] + enabled: bool = pydantic.Field(default=None) # pyrefly: ignore[bad-assignment] + job_type: JobTypeID = pydantic.Field(default=None) # pyrefly: ignore[bad-assignment] + job_parameters: dict[str, typing.Any] = pydantic.Field(default_factory=dict) + job_timeout_seconds: int | None = pydantic.Field(default=None, gt=0) diff --git a/inkcre_core/schemas/deployment_config.py b/inkcre_core/schemas/deployment_config.py new file mode 100644 index 00000000..0c4e4c61 --- /dev/null +++ b/inkcre_core/schemas/deployment_config.py @@ -0,0 +1,69 @@ +"""Deployment-scoped shared configuration persistence and API models.""" + +import datetime +import typing + +import pydantic +import sqlalchemy +import sqlalchemy.dialects.postgresql +import sqlmodel + + +DeploymentConfigKey: typing.TypeAlias = str +DeploymentConfigSchemaID: typing.TypeAlias = str + + +class DeploymentConfigModel(sqlmodel.SQLModel, table=True): + """One deployment-owned config value under an exact schema contract.""" + + __tablename__ = "configs" # type: ignore + + key: DeploymentConfigKey = sqlmodel.Field( + sa_column=sqlalchemy.Column(sqlalchemy.Text, primary_key=True) + ) + schema_id: DeploymentConfigSchemaID = sqlmodel.Field( + sa_column=sqlalchemy.Column("schema", sqlalchemy.Text, nullable=False) + ) + value: dict[str, typing.Any] = sqlmodel.Field( + sa_column=sqlalchemy.Column( + sqlalchemy.dialects.postgresql.JSONB, + nullable=False, + ) + ) + created_at: datetime.datetime = sqlmodel.Field( + default=None, + sa_column=sqlalchemy.Column( + sqlalchemy.TIMESTAMP(timezone=True), + nullable=False, + server_default=sqlalchemy.text("CURRENT_TIMESTAMP"), + ), + ) + updated_at: datetime.datetime = sqlmodel.Field( + default=None, + sa_column=sqlalchemy.Column( + sqlalchemy.TIMESTAMP(timezone=True), + nullable=False, + server_default=sqlalchemy.text("CURRENT_TIMESTAMP"), + ), + ) + + +class DeploymentConfigReplaceForm(pydantic.BaseModel): + """Complete PUT body; schema changes are allowed only through this form.""" + + model_config = pydantic.ConfigDict(populate_by_name=True) + + schema_id: DeploymentConfigSchemaID = pydantic.Field(alias="schema") + value: dict[str, typing.Any] + + +class DeploymentConfigView(pydantic.BaseModel): + """Stored deployment config and metadata; reading does not revalidate value.""" + + model_config = pydantic.ConfigDict(populate_by_name=True) + + key: DeploymentConfigKey + schema_id: DeploymentConfigSchemaID = pydantic.Field(alias="schema") + value: dict[str, typing.Any] + created_at: datetime.datetime + updated_at: datetime.datetime diff --git a/inkcre_core/schemas/extension/__init__.py b/inkcre_core/schemas/extension/__init__.py new file mode 100644 index 00000000..4a6fac60 --- /dev/null +++ b/inkcre_core/schemas/extension/__init__.py @@ -0,0 +1,37 @@ +from .main import ( + DisableExtensionCommand, + EnableExtensionCommand, + ExtensionPeerStatus, + ExtensionPeerStatusRow, + EXTENSION_NAME_PATTERN, + EXTENSION_SEMVER_PATTERN, + ExtensionManagementCommand, + ExtensionModel, + ExtensionPeerParticipationModel, + ExtensionName, + InstallExtensionCommand, + PatchExtensionConfigCommand, + SelectExtensionInput, + SelectExtensionCommand, + ExtensionStatusCommand, + UnselectExtensionCommand, +) + +__all__ = [ + "DisableExtensionCommand", + "EnableExtensionCommand", + "ExtensionPeerStatus", + "ExtensionPeerStatusRow", + "EXTENSION_NAME_PATTERN", + "EXTENSION_SEMVER_PATTERN", + "ExtensionManagementCommand", + "ExtensionModel", + "ExtensionPeerParticipationModel", + "ExtensionName", + "InstallExtensionCommand", + "PatchExtensionConfigCommand", + "SelectExtensionInput", + "SelectExtensionCommand", + "ExtensionStatusCommand", + "UnselectExtensionCommand", +] diff --git a/inkcre_core/schemas/extension/main.py b/inkcre_core/schemas/extension/main.py new file mode 100644 index 00000000..0d6322ef --- /dev/null +++ b/inkcre_core/schemas/extension/main.py @@ -0,0 +1,215 @@ +import typing +import uuid + +import pydantic +import sqlalchemy +import sqlalchemy.dialects.postgresql +import sqlmodel + + +ExtensionName: typing.TypeAlias = str +NativeDistribution = typing.Annotated[str, pydantic.Field(min_length=1)] +EXTENSION_NAME_PATTERN = ( + r"^[a-z0-9]([a-z0-9-]{0,62}[a-z0-9])?/" + r"[a-z0-9]([a-z0-9-]{0,62}[a-z0-9])?$" +) +EXTENSION_SEMVER_PATTERN = ( + r"^(0|[1-9][0-9]*)[.](0|[1-9][0-9]*)[.](0|[1-9][0-9]*)" + r"(-(0|[1-9][0-9]*|[0-9]*[A-Za-z-][0-9A-Za-z-]*)" + r"([.](0|[1-9][0-9]*|[0-9]*[A-Za-z-][0-9A-Za-z-]*))*)?$" +) + + +class InstallExtensionCommand(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + action: typing.Literal["install"] + extension: ExtensionName + version: str = pydantic.Field(pattern=EXTENSION_SEMVER_PATTERN) + + +class SelectExtensionCommand(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + action: typing.Literal["select"] + extension: ExtensionName + distribution: NativeDistribution + version: str = pydantic.Field(pattern=EXTENSION_SEMVER_PATTERN) + + +class ExtensionStatusCommand(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + action: typing.Literal["status"] + extension: ExtensionName | None = None + + +class UnselectExtensionCommand(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + action: typing.Literal["unselect"] + extension: ExtensionName + + +class EnableExtensionCommand(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + action: typing.Literal["enable"] + extension: ExtensionName + + +class DisableExtensionCommand(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + action: typing.Literal["disable"] + extension: ExtensionName + + +class PatchExtensionConfigCommand(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + action: typing.Literal["patch_config"] + extension: ExtensionName + patch: dict[str, typing.Any] + + +class ExtensionPeerStatusRow(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + name: ExtensionName + peer_id: uuid.UUID + distribution: NativeDistribution | None + version: str | None + enabled: bool + config_schema: dict[str, typing.Any] | None + supported_distributions: tuple[NativeDistribution, ...] + loaded_version: str | None + running: bool + restart_required: bool + operation: str | None + error: str | None + + +class ExtensionPeerStatus(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + peer_id: uuid.UUID + supported_distributions: tuple[NativeDistribution, ...] + extensions: tuple[ExtensionPeerStatusRow, ...] + + +class SelectExtensionInput(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + distribution: NativeDistribution + version: str = pydantic.Field(pattern=EXTENSION_SEMVER_PATTERN) + + +ExtensionManagementCommand: typing.TypeAlias = typing.Annotated[ + SelectExtensionCommand + | ExtensionStatusCommand + | UnselectExtensionCommand + | EnableExtensionCommand + | DisableExtensionCommand + | PatchExtensionConfigCommand, + pydantic.Field(discriminator="action"), +] + + +class ExtensionModel(sqlmodel.SQLModel, table=True): + """Shared Extension identity, configuration, and producer-owned state.""" + + __tablename__: str = "extensions" # type: ignore + __table_args__ = ( + sqlalchemy.CheckConstraint( + f"name ~ '{EXTENSION_NAME_PATTERN}'", + name="extensions_name_canonical", + ), + sqlalchemy.CheckConstraint( + "jsonb_typeof(state) = 'object'", + name="extensions_state_object", + ), + ) + + name: ExtensionName = sqlmodel.Field( + sa_column=sqlalchemy.Column(sqlalchemy.Text, primary_key=True) + ) + nickname: str | None = sqlmodel.Field( + default=None, + sa_column=sqlalchemy.Column(sqlalchemy.Text, nullable=True), + ) + config: dict = sqlmodel.Field( + default_factory=dict, + sa_column=sqlalchemy.Column( + sqlalchemy.dialects.postgresql.JSONB, + server_default=sqlalchemy.text("'{}'::jsonb"), + nullable=False, + ), + ) + state: dict = sqlmodel.Field( + default_factory=dict, + sa_column=sqlalchemy.Column( + sqlalchemy.dialects.postgresql.JSONB, + server_default=sqlalchemy.text("'{}'::jsonb"), + nullable=False, + ), + ) + + +class ExtensionPeerParticipationModel(sqlmodel.SQLModel, table=True): + """One Peer’s exact native distribution selection and enable intent.""" + + __tablename__: str = "extension_peer_participations" # type: ignore + __table_args__ = ( + sqlalchemy.CheckConstraint( + f"name ~ '{EXTENSION_NAME_PATTERN}'", + name="extension_peer_participations_name_canonical", + ), + sqlalchemy.CheckConstraint( + f"version ~ '{EXTENSION_SEMVER_PATTERN}'", + name="extension_peer_participations_version_canonical", + ), + sqlalchemy.CheckConstraint( + "config_schema IS NULL OR jsonb_typeof(config_schema) = 'object'", + name="extension_peer_participations_config_schema_object", + ), + sqlalchemy.CheckConstraint( + "distribution IS NOT NULL OR config_schema IS NULL", + name="extension_peer_participations_unresolved_schema_null", + ), + sqlalchemy.ForeignKeyConstraint( + ["name"], ["inkcre.extensions.name"], ondelete="CASCADE" + ), + sqlalchemy.ForeignKeyConstraint(["peer_id"], ["inkcre.peers.id"], ondelete="CASCADE"), + sqlalchemy.PrimaryKeyConstraint( + "name", "peer_id", name="extension_peer_participations_pkey" + ), + ) + + name: ExtensionName = sqlmodel.Field( + sa_column=sqlalchemy.Column(sqlalchemy.Text, nullable=False) + ) + peer_id: uuid.UUID = sqlmodel.Field( + sa_column=sqlalchemy.Column( + sqlalchemy.dialects.postgresql.UUID(as_uuid=True), nullable=False + ) + ) + distribution: str | None = sqlmodel.Field( + default=None, + sa_column=sqlalchemy.Column(sqlalchemy.Text, nullable=True), + ) + version: str = sqlmodel.Field( + sa_column=sqlalchemy.Column(sqlalchemy.Text, nullable=False) + ) + enabled: bool = sqlmodel.Field( + default=False, + sa_column=sqlalchemy.Column( + sqlalchemy.Boolean, nullable=False, server_default=sqlalchemy.false() + ), + ) + config_schema: dict | None = sqlmodel.Field( + default=None, + sa_column=sqlalchemy.Column( + sqlalchemy.dialects.postgresql.JSONB(none_as_null=True), nullable=True + ), + ) diff --git a/inkcre_core/schemas/graph_navigation_retrieval.py b/inkcre_core/schemas/graph_navigation_retrieval.py new file mode 100644 index 00000000..3555052f --- /dev/null +++ b/inkcre_core/schemas/graph_navigation_retrieval.py @@ -0,0 +1,90 @@ +"""Presentation-neutral graph-navigation retrieval contracts.""" + +import typing + +import pydantic + +from inkcre_core.schemas.info_base.block import BlockID, BlockModel +from inkcre_core.schemas.info_base.relation import RelationID, RelationModel + + +GraphDirection: typing.TypeAlias = typing.Literal["in", "out", "both"] + +DEFAULT_NEIGHBORHOOD_LIMIT = 20 +MAX_NEIGHBORHOOD_LIMIT = 100 +DEFAULT_MAX_HOPS = 4 +MAX_MAX_HOPS = 8 +DEFAULT_MAX_EXPLORED_BLOCKS = 1000 +MAX_MAX_EXPLORED_BLOCKS = 10000 +DEFAULT_MAX_EXPLORED_RELATIONS = 10000 + + +class GraphModel(pydantic.BaseModel): + """Endpoint-closed persisted graph read model.""" + + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + blocks: tuple[BlockModel, ...] + relations: tuple[RelationModel, ...] + + +class BlockNeighborhood(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + focal_block: BlockID + graph: GraphModel + next_cursor: RelationID | None = None + + +class RelationNeighborhood(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + focal_relation: RelationID + graph: GraphModel + + +class ConnectedSeedComponent(pydantic.BaseModel): + """One observed undirected component containing one or more input seeds.""" + + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + seed_block_ids: tuple[BlockID, ...] + member_block_ids: tuple[BlockID, ...] + + +class ConnectedComponentsResult(pydantic.BaseModel): + """Bounded seed partition plus an endpoint-closed spanning proof.""" + + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + components: tuple[ConnectedSeedComponent, ...] + proof_graph: GraphModel + missing_seed_block_ids: tuple[BlockID, ...] + truncated: bool + + +class PathFound(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + status: typing.Literal["found"] = "found" + graph: GraphModel + block_path: tuple[BlockID, ...] + relation_path: tuple[RelationID, ...] + + +class PathNotFound(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + status: typing.Literal["not_found"] = "not_found" + + +class PathLimitReached(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + status: typing.Literal["limit_reached"] = "limit_reached" + + +PathResult: typing.TypeAlias = typing.Annotated[ + PathFound | PathNotFound | PathLimitReached, + pydantic.Field(discriminator="status"), +] diff --git a/inkcre_core/schemas/info_base/__init__.py b/inkcre_core/schemas/info_base/__init__.py new file mode 100644 index 00000000..abcd96c5 --- /dev/null +++ b/inkcre_core/schemas/info_base/__init__.py @@ -0,0 +1,36 @@ +from .block import BlockForm, BlockModel, BlockID, ResolverType +from .relation import RelationCreateForm, RelationForm, RelationModel, RelationID +from .storage import StorageBlobModel, StorageModel, StorageID, StorageTypesModel +from .main import ( + GraphBlockForm, + GraphForm, + GraphRelationForm, + InArcForm, + OutArcForm, + StarsGraphForm, + SubmitGraphResult, + Vector, +) + +__all__ = [ + "BlockModel", + "BlockForm", + "BlockID", + "ResolverType", + "RelationModel", + "RelationForm", + "RelationCreateForm", + "RelationID", + "StorageModel", + "StorageBlobModel", + "StorageID", + "StorageTypesModel", + "StarsGraphForm", + "OutArcForm", + "InArcForm", + "GraphBlockForm", + "GraphRelationForm", + "GraphForm", + "SubmitGraphResult", + "Vector", +] diff --git a/inkcre_core/schemas/info_base/block.py b/inkcre_core/schemas/info_base/block.py new file mode 100644 index 00000000..83195023 --- /dev/null +++ b/inkcre_core/schemas/info_base/block.py @@ -0,0 +1,111 @@ +import datetime +import typing +import sqlalchemy +import sqlmodel +from pydantic import PrivateAttr +from typing import Optional as Opt +from .storage import StorageID + +ResolverType: typing.TypeAlias = str +"""Type of resolver. + +Extension resolvers should follows `extensions.{extension_id}.{resolver_name}` pattern. +""" +BlockID: typing.TypeAlias = int +HydratedContent: typing.TypeAlias = str | bytes +ResolverTV = typing.TypeVar("ResolverTV", bound="ResolverType") +"""Resolver type variable""" + + +class BlockForm(sqlmodel.SQLModel): + """Producer-owned values for creating one Block.""" + + model_config = {"extra": "forbid"} + + storage: Opt[StorageID] = sqlmodel.Field( + default=None, + sa_column=sqlalchemy.Column( + sqlalchemy.Integer, + sqlalchemy.ForeignKey("storages.id", onupdate="CASCADE", ondelete="RESTRICT"), + nullable=True, + ), + ) + resolver: ResolverType = sqlmodel.Field( + sa_column=sqlalchemy.Column(sqlalchemy.Text, nullable=False) + ) + content: str = sqlmodel.Field( + sa_column=sqlalchemy.Column(sqlalchemy.Text, nullable=False) + ) + """Content of the block. + + Block content stored as text in database. + For runtime usage, use storage to get the raw content. + """ + + +class BlockModel(BlockForm, table=True): + __tablename__ = "blocks" # type: ignore + + id: Opt[BlockID] = sqlmodel.Field( + sa_column=sqlmodel.Column(sqlmodel.Integer, primary_key=True, autoincrement=True), + default=None, + ) + created_at: datetime.datetime = sqlmodel.Field( + default=None, + sa_column=sqlalchemy.Column( + sqlalchemy.TIMESTAMP(timezone=True), + server_default=sqlalchemy.text("CURRENT_TIMESTAMP"), + ), + ) + updated_at: datetime.datetime = sqlmodel.Field( + default=None, + sa_column=sqlalchemy.Column( + sqlalchemy.TIMESTAMP(timezone=True), + server_default=sqlalchemy.text("CURRENT_TIMESTAMP"), + ), + ) + + _hydrated_content: HydratedContent | None = PrivateAttr(default=None) + _hydrated_content_source: tuple[StorageID | None, str] | None = PrivateAttr(default=None) + + async def get_hydrated_content(self, *, refresh: bool = False) -> HydratedContent: + """Return inline content or bytes loaded through the configured storage. + + The cache belongs only to this block instance. It is keyed by the persisted + storage pointer fields and is replaced when ``refresh`` is requested. + """ + source = (self.storage, self.content) + private_state = self.__pydantic_private__ + if private_state is None: + private_state = {} + object.__setattr__(self, "__pydantic_private__", private_state) + if ( + not refresh + and private_state.get("_hydrated_content_source") == source + and private_state.get("_hydrated_content") is not None + ): + return typing.cast(HydratedContent, private_state["_hydrated_content"]) + + if self.storage is None: + hydrated_content: HydratedContent = self.content + else: + from inkcre_core.business.info_base.storage import StorageManager + + storage = await StorageManager.get_storage_async(self.storage) + stored_content = await storage.get_raw_content(self.content) + if not isinstance(stored_content, bytes): + raise TypeError( + f"Storage {storage.__class__.__name__} returned " + f"{type(stored_content).__name__}; configured storage must return bytes" + ) + hydrated_content = stored_content + + private_state["_hydrated_content"] = hydrated_content + private_state["_hydrated_content_source"] = source + return hydrated_content + + async def get_context_as_text(self) -> str | None: + from inkcre_core.business.info_base.resolver import ResolverManager + + resolver = ResolverManager.get(self) + return await resolver.get_text() diff --git a/inkcre_core/schemas/info_base/main.py b/inkcre_core/schemas/info_base/main.py new file mode 100644 index 00000000..81e9bfb3 --- /dev/null +++ b/inkcre_core/schemas/info_base/main.py @@ -0,0 +1,116 @@ +"""Producer graph forms and graph-command results.""" + +__all__ = [ + "Vector", + "OutArcForm", + "InArcForm", + "StarsGraphForm", + "GraphBlockForm", + "GraphRelationForm", + "GraphForm", + "GraphBlockIDMapping", + "SubmitGraphResult", +] + +import typing + +import pydantic +import sqlmodel + +from inkcre_core.schemas.info_base.block import BlockForm, BlockID +from inkcre_core.schemas.info_base.relation import RelationForm + + +Vector: typing.TypeAlias = tuple[float, ...] +NegativeBlockID: typing.TypeAlias = typing.Annotated[int, pydantic.Field(lt=0)] +NonZeroBlockID: typing.TypeAlias = int + + +class OutArcForm(sqlmodel.SQLModel): + """One outgoing relation from the enclosing star to a recursive star.""" + + model_config = {"extra": "forbid"} + + relation: RelationForm + to_graph: "StarsGraphForm" + + +class InArcForm(sqlmodel.SQLModel): + """One incoming relation from a recursive star to the enclosing star.""" + + model_config = {"extra": "forbid"} + + relation: RelationForm + from_graph: "StarsGraphForm" + + +class StarsGraphForm(sqlmodel.SQLModel): + """Recursive star-centered authoring form used by Resolvers and extensions.""" + + model_config = {"extra": "forbid"} + + block: BlockForm + out_arcs: tuple[OutArcForm, ...] = () + in_arcs: tuple[InArcForm, ...] = () + + +class GraphBlockForm(BlockForm): + """A new Block declaration under one GraphForm-local negative ID.""" + + id: NegativeBlockID + + +class GraphRelationForm(RelationForm): + """A Relation declaration over the GraphForm signed Block-ID namespace.""" + + from_: NonZeroBlockID + to_: NonZeroBlockID + + @pydantic.field_validator("from_", "to_") + @classmethod + def require_non_zero_endpoint(cls, value: int) -> int: + if value == 0: + raise ValueError("GraphForm Relation endpoints must be non-zero") + return value + + +class GraphForm(sqlmodel.SQLModel): + """Flat command for adding arbitrarily connected Blocks and Relations.""" + + model_config = {"extra": "forbid"} + + blocks: tuple[GraphBlockForm, ...] = () + relations: tuple[GraphRelationForm, ...] = () + + @pydantic.model_validator(mode="after") + def validate_local_block_references(self) -> "GraphForm": + local_ids = [block.id for block in self.blocks] + if len(local_ids) != len(set(local_ids)): + raise ValueError("GraphForm new Block IDs must be unique") + + unresolved = { + endpoint + for relation in self.relations + for endpoint in (relation.from_, relation.to_) + if endpoint < 0 and endpoint not in local_ids + } + if unresolved: + rendered = ", ".join(str(value) for value in sorted(unresolved)) + raise ValueError(f"GraphForm contains unresolved local Block IDs: {rendered}") + return self + + +class GraphBlockIDMapping(sqlmodel.SQLModel): + """Map one command-local Block identity to its persisted identity.""" + + local_id: NegativeBlockID + id: BlockID + + +class SubmitGraphResult(sqlmodel.SQLModel): + """Minimal successful graph-command result used by producers and Agent Tools.""" + + blocks: tuple[GraphBlockIDMapping, ...] + + +StarsGraphForm.model_rebuild() diff --git a/inkcre_core/schemas/info_base/relation.py b/inkcre_core/schemas/info_base/relation.py new file mode 100644 index 00000000..506cffa2 --- /dev/null +++ b/inkcre_core/schemas/info_base/relation.py @@ -0,0 +1,68 @@ +import typing +from typing import Optional as Opt +import datetime +import sqlalchemy +import sqlmodel + +from inkcre_core.schemas.info_base.block import BlockID + +RelationID: typing.TypeAlias = int + + +class RelationForm(sqlmodel.SQLModel): + """Producer-owned values for creating one Relation.""" + + model_config = {"extra": "forbid"} + + content: str = sqlmodel.Field( + sa_column=sqlalchemy.Column(sqlalchemy.Text, nullable=False) + ) + + +class RelationCreateForm(RelationForm): + """Standalone Relation creation command with persisted Block endpoints.""" + + from_: BlockID + to_: BlockID + + +class RelationModel(RelationForm, table=True): + __tablename__ = "relations" # type: ignore + __table_args__ = ( + sqlalchemy.Index( + "relations_from_id_desc_idx", + "from_", + sqlalchemy.desc("id"), + ), + sqlalchemy.Index( + "relations_to_id_desc_idx", + "to_", + sqlalchemy.desc("id"), + ), + ) + + id: Opt[RelationID] = sqlmodel.Field( + sa_column=sqlmodel.Column(sqlmodel.Integer, primary_key=True, autoincrement=True), + default=None, + ) + updated_at: datetime.datetime = sqlmodel.Field( + default=None, + sa_column=sqlalchemy.Column( + sqlalchemy.TIMESTAMP(timezone=True), + server_default=sqlalchemy.text("CURRENT_TIMESTAMP"), + ), + ) + from_: int = sqlmodel.Field( + sa_column=sqlalchemy.Column( + sqlalchemy.Integer, + sqlalchemy.ForeignKey("blocks.id", ondelete="CASCADE", onupdate="CASCADE"), + ), + default=0, + ) + to_: int = sqlmodel.Field( + sa_column=sqlalchemy.Column( + sqlalchemy.Integer, + sqlalchemy.ForeignKey("blocks.id", ondelete="CASCADE", onupdate="CASCADE"), + ), + default=0, + ) diff --git a/inkcre_core/schemas/info_base/rest.py b/inkcre_core/schemas/info_base/rest.py new file mode 100644 index 00000000..4fb48a2e --- /dev/null +++ b/inkcre_core/schemas/info_base/rest.py @@ -0,0 +1,60 @@ +"""Ordinary REST input forms; persisted entities and Peer wire formats stay separate.""" + +import typing + +import pydantic + +from inkcre_core.schemas.graph_navigation_retrieval import ( + DEFAULT_MAX_HOPS, + MAX_MAX_HOPS, + DEFAULT_MAX_EXPLORED_BLOCKS, + MAX_MAX_EXPLORED_BLOCKS, + DEFAULT_MAX_EXPLORED_RELATIONS, + GraphDirection, +) + + +class EntityReference(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid") + type: typing.Literal["block", "relation"] + id: int + + +class EntitiesGetForm(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid") + entities: list[EntityReference] + content: typing.Literal["none", "raw", "hydrated"] = "raw" + + +class BlockUpdateForm(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid") + content: str = pydantic.Field(default=None) # pyrefly: ignore[bad-assignment] + resolver: str = pydantic.Field(default=None) # pyrefly: ignore[bad-assignment] + storage: int | None = None + + +class RelationUpdateForm(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid") + from_block_id: int = pydantic.Field(default=None) # pyrefly: ignore[bad-assignment] + to_block_id: int = pydantic.Field(default=None) # pyrefly: ignore[bad-assignment] + content: str = pydantic.Field(default=None) # pyrefly: ignore[bad-assignment] + + +class GraphPathForm(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid") + from_block_id: int + to_block_id: int + direction: GraphDirection = "both" + contents: tuple[str, ...] = () + max_hops: int = pydantic.Field(DEFAULT_MAX_HOPS, ge=0, le=MAX_MAX_HOPS) + max_explored_blocks: int = pydantic.Field( + DEFAULT_MAX_EXPLORED_BLOCKS, ge=1, le=MAX_MAX_EXPLORED_BLOCKS + ) + + +class GraphComponentsForm(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid") + seed_block_ids: tuple[int, ...] + contents: tuple[str, ...] = pydantic.Field(min_length=1) + max_explored_blocks: int = pydantic.Field(DEFAULT_MAX_EXPLORED_BLOCKS, ge=1) + max_explored_relations: int = pydantic.Field(DEFAULT_MAX_EXPLORED_RELATIONS, ge=1) diff --git a/inkcre_core/schemas/info_base/storage.py b/inkcre_core/schemas/info_base/storage.py new file mode 100644 index 00000000..4aeb5cc4 --- /dev/null +++ b/inkcre_core/schemas/info_base/storage.py @@ -0,0 +1,94 @@ +import typing +import uuid +import sqlalchemy +import sqlalchemy.dialects.postgresql +import sqlmodel +from typing import Optional as Opt + + +StorageTypeID: typing.TypeAlias = str +StorageID: typing.TypeAlias = int + + +class StorageBlobModel(sqlmodel.SQLModel, table=True): + """Raw PostgreSQL-owned bytes addressed by an opaque generated pointer.""" + + __tablename__: str = "storage_blobs" # type: ignore + + id: uuid.UUID = sqlmodel.Field( + default_factory=uuid.uuid4, + sa_column=sqlalchemy.Column( + sqlalchemy.dialects.postgresql.UUID(as_uuid=True), + primary_key=True, + server_default=sqlalchemy.text("gen_random_uuid()"), + ), + ) + data: bytes = sqlmodel.Field( + sa_column=sqlalchemy.Column(sqlalchemy.LargeBinary, nullable=False) + ) + + +class StorageTypesModel(sqlmodel.SQLModel, table=True): + __tablename__: str = "storage_types" # type: ignore + + id: StorageTypeID = sqlmodel.Field( + sa_column=sqlalchemy.Column(sqlalchemy.Text, primary_key=True) + ) + """Type of storage. + + A custom string to identify the storage type. + For extension storages, must follow `extensions.{extension_id}.{type}` format. + """ + description: str = sqlmodel.Field( + sa_column=sqlalchemy.Column(sqlalchemy.Text, nullable=False) + ) + """Description of this storage type.""" + config_schema: dict = sqlmodel.Field( + sa_column=sqlalchemy.Column( + sqlalchemy.dialects.postgresql.JSONB, + server_default=sqlalchemy.text("'{}'::jsonb"), + nullable=False, + ), + default=dict, + ) + writable: bool = sqlmodel.Field( + default=False, + sa_column=sqlalchemy.Column( + sqlalchemy.Boolean, + nullable=False, + server_default=sqlalchemy.false(), + ), + ) + + +class StorageModel(sqlmodel.SQLModel, table=True): + __tablename__: str = "storages" # type: ignore + + id: Opt[StorageID] = sqlmodel.Field( + sa_column=sqlalchemy.Column(sqlalchemy.Integer, primary_key=True, autoincrement=True), + default=None, + ) + type: str = sqlmodel.Field( + sa_column=sqlalchemy.Column( + sqlalchemy.Text, + sqlalchemy.ForeignKey("storage_types.id", onupdate="CASCADE", ondelete="CASCADE"), + nullable=False, + ) + ) + """Type of storage. + + An absolute import path to the module where storage class at. + When delete a storage type, all storages of this type will be deleted too. + """ + nickname: Opt[str] = sqlmodel.Field( + default=None, + sa_column=sqlalchemy.Column(sqlalchemy.Text, nullable=True), + ) + config: dict = sqlmodel.Field( + sa_column=sqlalchemy.Column( + sqlalchemy.dialects.postgresql.JSONB, + server_default=sqlalchemy.text("'{}'::jsonb"), + nullable=False, + ), + default=dict, + ) diff --git a/inkcre_core/schemas/job.py b/inkcre_core/schemas/job.py new file mode 100644 index 00000000..740c0a88 --- /dev/null +++ b/inkcre_core/schemas/job.py @@ -0,0 +1,173 @@ +"""Durable background-command schemas.""" + +import datetime +import enum +import typing + +import pydantic +import sqlalchemy +import sqlalchemy.dialects.postgresql +import sqlmodel + + +JobTypeID: typing.TypeAlias = str +JobID: typing.TypeAlias = int + + +class JobStatus(enum.StrEnum): + PENDING = "pending" + RUNNING = "running" + FINISHED = "finished" + FAILED = "failed" + TIMED_OUT = "timed_out" + ABORTED = "aborted" + + @property + def terminal(self) -> bool: + return self not in {self.PENDING, self.RUNNING} + + +class JobTypeModel(sqlmodel.SQLModel, table=True): + """Database projection of one exact runtime Job Handler contract.""" + + __tablename__ = "job_types" # type: ignore + __table_args__ = ( + sqlalchemy.CheckConstraint( + "default_timeout_seconds > 0", + name="job_types_default_timeout_seconds_positive", + ), + ) + + id: JobTypeID = sqlmodel.Field( + sa_column=sqlalchemy.Column(sqlalchemy.Text, primary_key=True) + ) + description: str = sqlmodel.Field( + sa_column=sqlalchemy.Column(sqlalchemy.Text, nullable=False) + ) + parameters_schema: dict[str, typing.Any] = sqlmodel.Field( + default_factory=dict, + sa_column=sqlalchemy.Column( + sqlalchemy.dialects.postgresql.JSONB, + nullable=False, + server_default=sqlalchemy.text("'{}'::jsonb"), + ), + ) + default_timeout_seconds: int = sqlmodel.Field( + gt=0, + sa_column=sqlalchemy.Column(sqlalchemy.Integer, nullable=False), + ) + + +class JobModel(sqlmodel.SQLModel, table=True): + """One durable execution opportunity for an exact registered command.""" + + __tablename__ = "jobs" # type: ignore + __table_args__ = ( + sqlalchemy.Index("jobs_status_idx", "status"), + sqlalchemy.CheckConstraint( + "octet_length(submission_traceparent) <= 512", + name="jobs_submission_traceparent_capacity", + ), + sqlalchemy.CheckConstraint( + "octet_length(submission_tracestate) <= 512", + name="jobs_submission_tracestate_capacity", + ), + sqlalchemy.CheckConstraint( + "timeout_seconds > 0", + name="jobs_timeout_seconds_positive", + ), + sqlalchemy.CheckConstraint( + "(status = 'pending' AND started_at IS NULL AND closed_at IS NULL) OR " + "(status = 'running' AND started_at IS NOT NULL AND closed_at IS NULL) OR " + "(status IN ('finished', 'failed', 'timed_out', 'aborted') " + "AND closed_at IS NOT NULL)", + name="jobs_lifecycle_timestamps_valid", + ), + ) + + id: JobID | None = sqlmodel.Field( + default=None, + sa_column=sqlalchemy.Column( + sqlalchemy.BigInteger, + sqlalchemy.Identity(), + primary_key=True, + ), + ) + type: JobTypeID = sqlmodel.Field( + sa_column=sqlalchemy.Column( + sqlalchemy.Text, + sqlalchemy.ForeignKey("job_types.id", onupdate="CASCADE", ondelete="RESTRICT"), + nullable=False, + ) + ) + parameters: dict[str, typing.Any] = sqlmodel.Field( + default_factory=dict, + sa_column=sqlalchemy.Column( + sqlalchemy.dialects.postgresql.JSONB, + nullable=False, + server_default=sqlalchemy.text("'{}'::jsonb"), + ), + ) + submission_traceparent: str | None = sqlmodel.Field( + default=None, + sa_column=sqlalchemy.Column(sqlalchemy.Text, nullable=True), + ) + submission_tracestate: str | None = sqlmodel.Field( + default=None, + sa_column=sqlalchemy.Column(sqlalchemy.Text, nullable=True), + ) + state: dict[str, typing.Any] = sqlmodel.Field( + default_factory=dict, + sa_column=sqlalchemy.Column( + sqlalchemy.dialects.postgresql.JSONB, + nullable=False, + server_default=sqlalchemy.text("'{}'::jsonb"), + ), + ) + abort_requested: bool = sqlmodel.Field( + default=False, + sa_column=sqlalchemy.Column( + sqlalchemy.Boolean, nullable=False, server_default=sqlalchemy.false() + ), + ) + timeout_seconds: int = sqlmodel.Field( + gt=0, + sa_column=sqlalchemy.Column(sqlalchemy.Integer, nullable=False), + ) + status: JobStatus = sqlmodel.Field( + default=JobStatus.PENDING, + sa_column=sqlalchemy.Column( + sqlalchemy.Enum( + JobStatus, + name="jobstatus", + values_callable=lambda values: [value.value for value in values], + inherit_schema=True, + ), + nullable=False, + server_default=JobStatus.PENDING.value, + ), + ) + created_at: datetime.datetime = sqlmodel.Field( + default=None, + sa_column=sqlalchemy.Column( + sqlalchemy.TIMESTAMP(timezone=True), + nullable=False, + server_default=sqlalchemy.text("CURRENT_TIMESTAMP"), + ), + ) + started_at: datetime.datetime | None = sqlmodel.Field( + default=None, + sa_column=sqlalchemy.Column(sqlalchemy.TIMESTAMP(timezone=True), nullable=True), + ) + closed_at: datetime.datetime | None = sqlmodel.Field( + default=None, + sa_column=sqlalchemy.Column(sqlalchemy.TIMESTAMP(timezone=True), nullable=True), + ) + + +class JobCreateForm(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid") + + type: JobTypeID + parameters: dict[str, typing.Any] = pydantic.Field(default_factory=dict) + timeout_seconds: int | None = pydantic.Field(default=None, gt=0) diff --git a/inkcre_core/schemas/lexical_retrieval.py b/inkcre_core/schemas/lexical_retrieval.py new file mode 100644 index 00000000..15d85e67 --- /dev/null +++ b/inkcre_core/schemas/lexical_retrieval.py @@ -0,0 +1,150 @@ +"""Block-local lexical projection, maintenance, and ranked result contracts.""" + +import datetime +import math +import typing + +import pydantic +import sqlalchemy +import sqlalchemy.dialects.postgresql +import sqlmodel + +from inkcre_core.schemas.info_base.block import BlockModel + + +LexicalEvidence: typing.TypeAlias = typing.Literal[ + "label_exact", + "label_substring", + "text_substring", + "terms", +] + + +class BlockLexicalRecordModel(sqlmodel.SQLModel, table=True): + """Rebuildable Resolver projection owned only by lexical retrieval.""" + + __tablename__ = "block_lexical_records" # type: ignore + __table_args__ = ( + sqlalchemy.Index( + "block_lexical_records_search_vector_idx", + "search_vector", + postgresql_using="gin", + ), + sqlalchemy.Index( + "block_lexical_records_label_trgm_idx", + "label", + postgresql_using="gin", + postgresql_ops={"label": "gin_trgm_ops"}, + ), + sqlalchemy.Index( + "block_lexical_records_text_trgm_idx", + "text", + postgresql_using="gin", + postgresql_ops={"text": "gin_trgm_ops"}, + ), + ) + + block: int = sqlmodel.Field( + sa_column=sqlalchemy.Column( + sqlalchemy.Integer, + sqlalchemy.ForeignKey("blocks.id", onupdate="CASCADE", ondelete="CASCADE"), + primary_key=True, + ) + ) + label: str = sqlmodel.Field(sa_column=sqlalchemy.Column(sqlalchemy.Text, nullable=False)) + text: str | None = sqlmodel.Field( + default=None, + sa_column=sqlalchemy.Column(sqlalchemy.Text, nullable=True), + ) + search_vector: str = sqlmodel.Field( + sa_column=sqlalchemy.Column( + sqlalchemy.dialects.postgresql.TSVECTOR, + nullable=False, + ) + ) + created_at: datetime.datetime = sqlmodel.Field( + default=None, + sa_column=sqlalchemy.Column( + sqlalchemy.TIMESTAMP(timezone=True), + nullable=False, + server_default=sqlalchemy.text("CURRENT_TIMESTAMP"), + ), + ) + updated_at: datetime.datetime = sqlmodel.Field( + default=None, + sa_column=sqlalchemy.Column( + sqlalchemy.TIMESTAMP(timezone=True), + nullable=False, + server_default=sqlalchemy.text("CURRENT_TIMESTAMP"), + ), + ) + + +class LexicalRetrievalRequest(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + query: str + limit: int = pydantic.Field(default=20, ge=1, le=20) + + @pydantic.field_validator("query") + @classmethod + def non_empty_query(cls, value: str) -> str: + if not value.strip(): + raise ValueError("query must not be empty") + return value + + +class LexicalRetrievalMatch(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + block: BlockModel + label: str + excerpt: str + evidence: LexicalEvidence + rank: float = pydantic.Field(ge=0) + + @pydantic.field_validator("rank") + @classmethod + def finite_rank(cls, value: float) -> float: + if not math.isfinite(value): + raise ValueError("rank must be finite") + return value + + +class LexicalRetrievalResult(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + matches: tuple[LexicalRetrievalMatch, ...] + + +class LexicalMaintenanceOptions(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + max_records: int = pydantic.Field(default=100, ge=1) + scan_page_size: int = pydantic.Field(default=100, ge=1) + diagnostic_limit: int = pydantic.Field(default=20, ge=0) + + +class LexicalMaintenanceDiagnostic(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + block: int + outcome: typing.Literal["unavailable", "failed"] + reason: str + + +class LexicalMaintenanceReport(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + indexed: int = 0 + unavailable: int = 0 + failed: int = 0 + diagnostics: tuple[LexicalMaintenanceDiagnostic, ...] = () + + +class LexicalMaintenanceJobParameters(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + options: LexicalMaintenanceOptions = pydantic.Field( + default_factory=LexicalMaintenanceOptions + ) diff --git a/inkcre_core/schemas/observability.py b/inkcre_core/schemas/observability.py new file mode 100644 index 00000000..b8e588ad --- /dev/null +++ b/inkcre_core/schemas/observability.py @@ -0,0 +1,37 @@ +"""Non-secret deployment telemetry configuration shared by admitted Peers.""" + +import uuid +import pydantic + +CONFIG_KEY = "inkcre.observability" +CONFIG_SCHEMA = "inkcre.observability.v1" + + +def _credential_free_url(value: pydantic.AnyHttpUrl | None): + if value is not None and ( + value.username or value.password or value.query or value.fragment + ): + raise ValueError("observability URLs cannot contain credentials, queries or fragments") + return value + + +class TelemetryEndpoints(pydantic.BaseModel): + """Complete OTLP/HTTP destinations; authentication stays in local runtime config.""" + + model_config = pydantic.ConfigDict(extra="forbid") + traces: pydantic.AnyHttpUrl | None = None + logs: pydantic.AnyHttpUrl | None = None + metrics: pydantic.AnyHttpUrl | None = None + + _urls = pydantic.field_validator("traces", "logs", "metrics")(_credential_free_url) + + +class ObservabilityConfig(pydantic.BaseModel): + """Stable deployment identity and optional metadata-only diagnostic destinations.""" + + model_config = pydantic.ConfigDict(extra="forbid") + deployment_id: uuid.UUID + otlp_http_endpoints: TelemetryEndpoints | None = None + diagnostics_url: pydantic.AnyHttpUrl | None = None + + _url = pydantic.field_validator("diagnostics_url")(_credential_free_url) diff --git a/inkcre_core/schemas/organization.py b/inkcre_core/schemas/organization.py new file mode 100644 index 00000000..63212028 --- /dev/null +++ b/inkcre_core/schemas/organization.py @@ -0,0 +1,89 @@ +"""Organization rumination configuration, inbound, and Agent Tool forms.""" + +import typing + +import pydantic + +from inkcre_core.schemas.ai import JSONValue +from inkcre_core.schemas.info_base.block import BlockID +from inkcre_core.schemas.info_base.main import GraphForm, NegativeBlockID + + +class RuminationConfig(pydantic.BaseModel): + """Deployment selection of one reusable Agent definition.""" + + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + agent: int + + +class MediaInterpretationConfig(pydantic.BaseModel): + """Deployment selection of independent modality-oriented Agents.""" + + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + image_agent: int + audio_agent: int + video_agent: int + + +class MediaInterpretationJobParameters(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + +class MediaInterpretationDiagnostic(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + block: int + modality: typing.Literal["image", "audio", "video"] + outcome: typing.Literal["unavailable", "failed", "no_output"] + reason: str + + +class MediaInterpretationReport(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + selected: int = 0 + interpreted: int = 0 + unavailable: int = 0 + failed: int = 0 + no_output: int = 0 + diagnostics: tuple[MediaInterpretationDiagnostic, ...] = () + + +class RuminationRequest(pydantic.BaseModel): + """Explicit request to reconsider one focal Block.""" + + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + block: BlockID + + +class GetDraftGraphSchemaInput(pydantic.BaseModel): + """Generic schema-discovery input narrowed when an Agent run binds Tools.""" + + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + resolver_types: tuple[str, ...] + + +class DraftGraphInput(pydantic.BaseModel): + """Generic draft request; Resolver input is validated by its bound run model.""" + + model_config = pydantic.ConfigDict(extra="forbid") + + resolver_type: str + input: dict[str, JSONValue] = pydantic.Field( + description="Arguments matching the selected Resolver's input_schema." + ) + local_block_id_start: NegativeBlockID = pydantic.Field( + default=-1, description="First temporary ID; keep IDs disjoint when combining drafts." + ) + + +class SubmitGraphInput(pydantic.BaseModel): + """The sole rumination Tool input that can mutate the info-base graph.""" + + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + graph: GraphForm diff --git a/inkcre_core/schemas/organization_behavior.py b/inkcre_core/schemas/organization_behavior.py new file mode 100644 index 00000000..42314738 --- /dev/null +++ b/inkcre_core/schemas/organization_behavior.py @@ -0,0 +1,146 @@ +"""Contracts for exact Organization behaviors and their Agent adapters.""" + +import typing + +import pydantic + +from inkcre_core.schemas.graph_navigation_retrieval import ( + GraphModel, +) +from inkcre_core.schemas.info_base.block import BlockID, ResolverType +from inkcre_core.schemas.info_base.relation import RelationID + + +class BehaviorAgentConfig(pydantic.BaseModel): + """Deployment selection of one purpose-built Agent definition.""" + + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + agent: int + + +class AutomaticOrganizationJobParameters(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + max_seeds: int = pydantic.Field(default=10, ge=3, le=100) + + +class RelationWriteResult(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + relation_id: RelationID + created: bool + + +class CandidateWriteResult(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + descriptor_block_id: BlockID + relation_id: RelationID + created: bool + + +class SupersessionProposal(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + successor_block_id: BlockID + predecessor_block_id: BlockID + + +class RefinementProposal(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + refinement_block_id: BlockID + predecessor_block_id: BlockID + + +class EvidenceStanceProposal(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + evidence_block_id: BlockID + assertion_block_id: BlockID + stance: typing.Literal["supports", "challenges"] + + +class SynthesisProposal(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + text: str + source_block_ids: tuple[BlockID, ...] = pydantic.Field( + min_length=2, description="Actual contributing sources, not all inspected context." + ) + previous_synthesis_block_id: BlockID | None = pydantic.Field( + default=None, description="Earlier synthesis revised through an edited relation." + ) + + @pydantic.field_validator("text") + @classmethod + def non_empty_text(cls, value: str) -> str: + if not value.strip(): + raise ValueError("text must not be empty") + return value + + @pydantic.field_validator("source_block_ids") + @classmethod + def distinct_sources(cls, value: tuple[BlockID, ...]) -> tuple[BlockID, ...]: + if len(set(value)) != len(value): + raise ValueError("source_block_ids must be distinct") + return value + + +class SynthesisWriteResult(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + synthesis_block_id: BlockID + synthesis_created: bool + basis: tuple[RelationWriteResult, ...] + edited: RelationWriteResult | None = None + + +class ExistingReferentAnchorProposal(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + source_block_id: BlockID + selected_text: str = pydantic.Field( + description="Minimal sufficient referring fragment from the source." + ) + referent_block_id: BlockID + + @pydantic.field_validator("selected_text") + @classmethod + def non_empty_selected_text(cls, value: str) -> str: + if not value.strip(): + raise ValueError("selected_text must not be empty") + return value + + +class ExistingReferentAnchorResult(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + fragment_block_id: BlockID + fragment_created: bool + has_mention: RelationWriteResult | None + refers_to: RelationWriteResult + + +class DuplicateAssertionProposal(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + left_block_id: BlockID + right_block_id: BlockID + + +class RecordOrganizationCandidateInput(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + block_id: BlockID + behavior: ResolverType + + +class SupersessionLineage(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + graph: GraphModel + current_block_ids: tuple[BlockID, ...] + truncated: bool + cycle_detected: bool diff --git a/inkcre_core/schemas/peer/__init__.py b/inkcre_core/schemas/peer/__init__.py new file mode 100644 index 00000000..6c1618f4 --- /dev/null +++ b/inkcre_core/schemas/peer/__init__.py @@ -0,0 +1,35 @@ +from .main import ( + CapabilityID, + CorePeerConfig, + PeerCapabilityAdvertisement, + PeerInboundInterface, + PeerModel, + PeerProtocolID, + PeerRef, + normalize_capability_snapshot, +) +from .protocol import ( + PEER_EXECUTION_HEADER, + PEER_HTTP_PROTOCOL, + PEER_NOT_EXECUTED, + PeerHTTPInboundParameters, + PeerProtocolRequest, + PeerProtocolResponse, +) + +__all__ = [ + "CapabilityID", + "CorePeerConfig", + "PEER_EXECUTION_HEADER", + "PEER_HTTP_PROTOCOL", + "PEER_NOT_EXECUTED", + "PeerCapabilityAdvertisement", + "PeerHTTPInboundParameters", + "PeerInboundInterface", + "PeerModel", + "PeerProtocolID", + "PeerProtocolRequest", + "PeerProtocolResponse", + "PeerRef", + "normalize_capability_snapshot", +] diff --git a/inkcre_core/schemas/peer/main.py b/inkcre_core/schemas/peer/main.py new file mode 100644 index 00000000..d71db6db --- /dev/null +++ b/inkcre_core/schemas/peer/main.py @@ -0,0 +1,178 @@ +"""Peer identity, configuration, capability advertisement, and persistence.""" + +import datetime +import typing +from urllib.parse import urlsplit, urlunsplit +import uuid + +import pydantic +import sqlalchemy +import sqlalchemy.dialects.postgresql +import sqlmodel + +from inkcre_core.schemas.ai import JSONValue + + +PeerRef: typing.TypeAlias = uuid.UUID +CapabilityID: typing.TypeAlias = str +PeerProtocolID: typing.TypeAlias = str + + +class PeerInboundInterface(pydantic.BaseModel): + """One protocol-discriminated construction descriptor for a caller outbound.""" + + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + protocol: PeerProtocolID = pydantic.Field(min_length=1) + parameters: dict[str, JSONValue] + + +class PeerCapabilityAdvertisement(pydantic.BaseModel): + """One exact capability and its provider-owned inbound interface.""" + + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + id: CapabilityID = pydantic.Field(min_length=1) + inbound: PeerInboundInterface + + +_CAPABILITIES_ADAPTER = pydantic.TypeAdapter(tuple[PeerCapabilityAdvertisement, ...]) + + +def normalize_capability_snapshot( + value: typing.Any, +) -> tuple[PeerCapabilityAdvertisement, ...]: + """Validate, de-duplicate, and deterministically order one full snapshot.""" + capabilities = _CAPABILITIES_ADAPTER.validate_python(value or ()) + ids = tuple(capability.id for capability in capabilities) + if len(ids) != len(set(ids)): + raise ValueError("Peer capability IDs must be unique") + return tuple(sorted(capabilities, key=lambda capability: capability.id)) + + +class CorePeerConfig(pydantic.BaseModel): + """impl-py-owned values stored in this Peer row's config object.""" + + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + http_public_base_url: str | None = pydantic.Field(default=None, title="Public HTTP URL") + extension_registry_url: str | None = pydantic.Field( + default=None, title="Extension Registry URL" + ) + + @pydantic.field_validator("http_public_base_url") + @classmethod + def valid_public_http_base(cls, value: str | None) -> str | None: + if value is None: + return None + parts = urlsplit(value) + if ( + parts.scheme not in {"http", "https"} + or not parts.netloc + or parts.username is not None + or parts.password is not None + or parts.query + or parts.fragment + ): + raise ValueError( + "http_public_base_url must be an absolute HTTP(S) URL without " + "credentials, query, or fragment" + ) + path = parts.path.rstrip("/") + return urlunsplit((parts.scheme, parts.netloc, path, "", "")) + + @pydantic.field_validator("extension_registry_url") + @classmethod + def valid_extension_registry_origin(cls, value: str | None) -> str | None: + if value is None or not value.strip(): + return None + parts = urlsplit(value.strip()) + if ( + parts.scheme not in {"http", "https"} + or not parts.netloc + or parts.username is not None + or parts.password is not None + or parts.path not in {"", "/"} + or parts.query + or parts.fragment + ): + raise ValueError("extension_registry_url must be one HTTP(S) origin") + return urlunsplit((parts.scheme, parts.netloc, "", "", "")) + + +class PeerModel(sqlmodel.SQLModel, table=True): + """One equal deployment Peer with a runtime-owned capability/lease snapshot.""" + + __tablename__ = "peers" # type: ignore + + id: PeerRef = sqlmodel.Field( + sa_column=sqlalchemy.Column( + sqlalchemy.dialects.postgresql.UUID(as_uuid=True), + primary_key=True, + default=uuid.uuid4, + ), + ) + name: str = sqlmodel.Field(sa_column=sqlalchemy.Column(sqlalchemy.Text, nullable=False)) + application_version: str | None = sqlmodel.Field( + default=None, + sa_column=sqlalchemy.Column(sqlalchemy.Text, nullable=True), + ) + labels: list[str] = sqlmodel.Field( + default_factory=list, + sa_column=sqlalchemy.Column( + sqlalchemy.dialects.postgresql.ARRAY(sqlalchemy.Text), + nullable=False, + server_default=sqlalchemy.text("'{}'::text[]"), + ), + ) + config: dict[str, JSONValue] = sqlmodel.Field( + default_factory=dict, + sa_column=sqlalchemy.Column( + sqlalchemy.dialects.postgresql.JSONB, + nullable=False, + server_default=sqlalchemy.text("'{}'::jsonb"), + ), + ) + config_schema: dict[str, JSONValue] = sqlmodel.Field( + default_factory=dict, + sa_column=sqlalchemy.Column( + sqlalchemy.dialects.postgresql.JSONB, + nullable=False, + server_default=sqlalchemy.text("'{}'::jsonb"), + ), + ) + capabilities: list[dict[str, JSONValue]] = sqlmodel.Field( + default_factory=list, + sa_column=sqlalchemy.Column( + sqlalchemy.dialects.postgresql.JSONB, + nullable=False, + server_default=sqlalchemy.text("'[]'::jsonb"), + ), + ) + lease_expires_at: datetime.datetime | None = sqlmodel.Field( + default=None, + sa_column=sqlalchemy.Column( + sqlalchemy.TIMESTAMP(timezone=True), + nullable=True, + ), + ) + created_at: datetime.datetime = sqlmodel.Field( + default=None, + sa_column=sqlalchemy.Column( + sqlalchemy.TIMESTAMP(timezone=True), + nullable=False, + server_default=sqlalchemy.text("CURRENT_TIMESTAMP"), + ), + ) + updated_at: datetime.datetime = sqlmodel.Field( + default=None, + sa_column=sqlalchemy.Column( + sqlalchemy.TIMESTAMP(timezone=True), + nullable=False, + server_default=sqlalchemy.text("CURRENT_TIMESTAMP"), + ), + ) + + def capability_snapshot(self) -> tuple[PeerCapabilityAdvertisement, ...]: + """Validate a possibly externally edited persisted snapshot at use time.""" + return _CAPABILITIES_ADAPTER.validate_python(self.capabilities or ()) diff --git a/inkcre_core/schemas/peer/protocol.py b/inkcre_core/schemas/peer/protocol.py new file mode 100644 index 00000000..4a7f7441 --- /dev/null +++ b/inkcre_core/schemas/peer/protocol.py @@ -0,0 +1,99 @@ +"""Normalized Peer HTTP v1 request/response envelopes.""" + +import re +import typing +from urllib.parse import urlsplit + +import pydantic + +from inkcre_core.schemas.ai import JSONValue + + +PEER_HTTP_PROTOCOL = "core.peer.protocol.http.v1" +PEER_EXECUTION_HEADER = "InkCre-Peer-Execution" +PEER_NOT_EXECUTED = "not-executed" + +_HTTP_TOKEN = re.compile(r"^[!#$%&'*+.^_`|~0-9A-Za-z-]+$") + + +def _normalized_multimap(value: typing.Any) -> dict[str, tuple[str, ...]]: + if value is None: + return {} + if not isinstance(value, dict): + raise ValueError("normalized map must be an object") + normalized: dict[str, tuple[str, ...]] = {} + for raw_name, raw_values in value.items(): + if not isinstance(raw_name, str) or not raw_name: + raise ValueError("normalized map names must be non-empty strings") + name = raw_name.lower() + if name in normalized: + raise ValueError(f"normalized map contains duplicate name: {name}") + if not isinstance(raw_values, (list, tuple)) or not all( + isinstance(item, str) for item in raw_values + ): + raise ValueError("normalized map values must be string arrays") + normalized[name] = tuple(raw_values) + return normalized + + +class PeerProtocolRequest(pydantic.BaseModel): + """One protocol payload; query, headers, and JSON body may coexist.""" + + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + query: dict[str, tuple[str, ...]] = pydantic.Field(default_factory=dict) + headers: dict[str, tuple[str, ...]] = pydantic.Field(default_factory=dict) + body: JSONValue = None + + @pydantic.field_validator("query", "headers", mode="before") + @classmethod + def normalized_maps(cls, value: typing.Any) -> dict[str, tuple[str, ...]]: + return _normalized_multimap(value) + + +class PeerProtocolResponse(pydantic.BaseModel): + """One executed HTTP response projected back into the normalized protocol.""" + + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + status: int = pydantic.Field(ge=100, le=599) + headers: dict[str, tuple[str, ...]] = pydantic.Field(default_factory=dict) + body: JSONValue = None + + @pydantic.field_validator("headers", mode="before") + @classmethod + def normalized_headers(cls, value: typing.Any) -> dict[str, tuple[str, ...]]: + return _normalized_multimap(value) + + +class PeerHTTPInboundParameters(pydantic.BaseModel): + """Static parameters published by one HTTP business inbound.""" + + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + method: str + url: str + + @pydantic.field_validator("method") + @classmethod + def valid_method(cls, value: str) -> str: + method = value.upper() + if not _HTTP_TOKEN.fullmatch(method): + raise ValueError("HTTP method must be one valid token") + return method + + @pydantic.field_validator("url") + @classmethod + def valid_absolute_url(cls, value: str) -> str: + parts = urlsplit(value) + if ( + parts.scheme not in {"http", "https"} + or not parts.netloc + or parts.username is not None + or parts.password is not None + or parts.fragment + ): + raise ValueError( + "Peer HTTP inbound URL must be absolute HTTP(S) without credentials or fragment" + ) + return value diff --git a/inkcre_core/schemas/semantic_retrieval.py b/inkcre_core/schemas/semantic_retrieval.py new file mode 100644 index 00000000..bcfef09c --- /dev/null +++ b/inkcre_core/schemas/semantic_retrieval.py @@ -0,0 +1,129 @@ +"""Semantic retrieval configuration, maintenance and ranked result contracts.""" + +import math +import typing + +import pydantic + +from inkcre_core.schemas.ai import EmbeddingProfileID +from inkcre_core.schemas.info_base.block import BlockModel +from inkcre_core.schemas.info_base.relation import RelationModel + + +SemanticEntityType: typing.TypeAlias = typing.Literal["block", "relation"] + + +def _default_entity_types() -> frozenset[SemanticEntityType]: + return frozenset(("block", "relation")) + + +class SemanticRetrievalConfig(pydantic.BaseModel): + """Deployment-owned selection of the default vector space.""" + + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + default_profile: EmbeddingProfileID | None = None + + +class VectorRetrievalOptions(pydantic.BaseModel): + """Query-scoped controls over one already compatible vector space.""" + + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + limit: int = pydantic.Field(default=20, ge=1, le=100) + min_score: float | None = pydantic.Field(default=None, ge=-1, le=1) + entity_types: frozenset[SemanticEntityType] = pydantic.Field( + default_factory=_default_entity_types, + min_length=1, + ) + + @pydantic.field_validator("min_score") + @classmethod + def finite_min_score(cls, value: float | None) -> float | None: + if value is not None and not math.isfinite(value): + raise ValueError("min_score must be finite") + return value + + +class SemanticRetrievalRequest(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + query: str + profile: EmbeddingProfileID | None = None + options: VectorRetrievalOptions = pydantic.Field(default_factory=VectorRetrievalOptions) + + @pydantic.field_validator("query") + @classmethod + def non_empty_query(cls, value: str) -> str: + if not value.strip(): + raise ValueError("query must not be empty") + return value + + +class BlockSemanticRetrievalMatch(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + type: typing.Literal["block"] = "block" + entity: BlockModel + score: float = pydantic.Field(ge=-1, le=1) + + +class RelationSemanticRetrievalMatch(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + type: typing.Literal["relation"] = "relation" + entity: RelationModel + score: float = pydantic.Field(ge=-1, le=1) + + +SemanticRetrievalMatch: typing.TypeAlias = typing.Annotated[ + BlockSemanticRetrievalMatch | RelationSemanticRetrievalMatch, + pydantic.Field(discriminator="type"), +] + + +class SemanticRetrievalResult(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + profile: EmbeddingProfileID + metric: typing.Literal["cosine"] = "cosine" + matches: tuple[SemanticRetrievalMatch, ...] + + +class EmbeddingMaintenanceOptions(pydantic.BaseModel): + """Peer-local bounds for one resumable maintenance invocation.""" + + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + max_embeddings: int = pydantic.Field(default=100, ge=1) + batch_size: int = pydantic.Field(default=20, ge=1) + scan_page_size: int = pydantic.Field(default=100, ge=1) + diagnostic_limit: int = pydantic.Field(default=20, ge=0) + + +class EmbeddingMaintenanceDiagnostic(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + entity_type: SemanticEntityType + entity: int + outcome: typing.Literal["unavailable", "failed"] + reason: str + + +class EmbeddingMaintenanceReport(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + profile: EmbeddingProfileID + embedded: int = 0 + unavailable: int = 0 + failed: int = 0 + diagnostics: tuple[EmbeddingMaintenanceDiagnostic, ...] = () + + +class EmbeddingMaintenanceJobParameters(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + profile: EmbeddingProfileID | None = None + options: EmbeddingMaintenanceOptions = pydantic.Field( + default_factory=EmbeddingMaintenanceOptions + ) diff --git a/inkcre_core/schemas/sink.py b/inkcre_core/schemas/sink.py new file mode 100644 index 00000000..ff3739be --- /dev/null +++ b/inkcre_core/schemas/sink.py @@ -0,0 +1,82 @@ +"""Persisted Sink catalog and instance contracts.""" + +import typing +import uuid + +import pydantic +import sqlalchemy +import sqlalchemy.dialects.postgresql +import sqlmodel + + +SinkTypeID: typing.TypeAlias = str +SinkID: typing.TypeAlias = int + + +class SinkTypeModel(sqlmodel.SQLModel, table=True): + __tablename__ = "sink_types" # type: ignore + + id: SinkTypeID = sqlmodel.Field( + sa_column=sqlalchemy.Column(sqlalchemy.Text, primary_key=True) + ) + description: str = sqlmodel.Field( + sa_column=sqlalchemy.Column(sqlalchemy.Text, nullable=False) + ) + config_schema: dict[str, typing.Any] = sqlmodel.Field( + default_factory=dict, + sa_column=sqlalchemy.Column( + sqlalchemy.dialects.postgresql.JSONB, + nullable=False, + server_default=sqlalchemy.text("'{}'::jsonb"), + ), + ) + + +class SinkModel(sqlmodel.SQLModel, table=True): + __tablename__ = "sinks" # type: ignore + + id: SinkID | None = sqlmodel.Field( + default=None, + sa_column=sqlalchemy.Column( + sqlalchemy.BigInteger, + sqlalchemy.Identity(), + primary_key=True, + ), + ) + type: SinkTypeID = sqlmodel.Field( + sa_column=sqlalchemy.Column( + sqlalchemy.Text, + sqlalchemy.ForeignKey("sink_types.id", onupdate="CASCADE", ondelete="RESTRICT"), + nullable=False, + ) + ) + nickname: str | None = sqlmodel.Field( + default=None, + sa_column=sqlalchemy.Column(sqlalchemy.Text, nullable=True), + ) + config: dict[str, typing.Any] = sqlmodel.Field( + default_factory=dict, + sa_column=sqlalchemy.Column( + sqlalchemy.dialects.postgresql.JSONB, + nullable=False, + server_default=sqlalchemy.text("'{}'::jsonb"), + ), + ) + enabled: list[uuid.UUID] = sqlmodel.Field( + default_factory=list, + sa_column=sqlalchemy.Column( + sqlalchemy.dialects.postgresql.ARRAY( + sqlalchemy.dialects.postgresql.UUID(as_uuid=True) + ), + nullable=False, + server_default=sqlalchemy.text("'{}'::uuid[]"), + ), + ) + + +class SinkCreateForm(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + type: SinkTypeID + nickname: str | None = None + config: dict[str, typing.Any] = pydantic.Field(default_factory=dict) diff --git a/inkcre_core/schemas/source/__init__.py b/inkcre_core/schemas/source/__init__.py new file mode 100644 index 00000000..a400ba16 --- /dev/null +++ b/inkcre_core/schemas/source/__init__.py @@ -0,0 +1,19 @@ +__all__ = [ + "SourceID", + "SourceModel", + "SourceTypesModel", + "SourceCollectParameters", + "SourceBackfillParameters", + "SourceCreateForm", + "SourceUpdateForm", +] + +from .main import ( + SourceBackfillParameters, + SourceCollectParameters, + SourceID, + SourceModel, + SourceTypesModel, + SourceCreateForm, + SourceUpdateForm, +) diff --git a/inkcre_core/schemas/source/main.py b/inkcre_core/schemas/source/main.py new file mode 100644 index 00000000..2bc109fd --- /dev/null +++ b/inkcre_core/schemas/source/main.py @@ -0,0 +1,186 @@ +__all__ = [ + "SourceID", + "SourceModel", + "SourceTypesModel", + "SourceCollectParameters", + "SourceBackfillParameters", +] + +import datetime +import pydantic +import sqlalchemy +import sqlalchemy.dialects +import sqlalchemy.dialects.postgresql +import sqlmodel +import typing +from typing import Optional as Opt + +from inkcre_core.schemas.info_base.block import BlockID +from inkcre_core.schemas.info_base.storage import StorageID + +SourceID: typing.TypeAlias = int + + +class SourceCreateForm(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid") + + type: str + nickname: str | None = None + config: dict = pydantic.Field(default_factory=dict) + storage: StorageID | None = None + + +class SourceUpdateForm(pydantic.BaseModel): + """Only submitted fields are written; config is replaced, not deep-merged.""" + + model_config = pydantic.ConfigDict(extra="forbid") + + nickname: str | None = None + config: dict = pydantic.Field(default_factory=dict) + storage: StorageID | None = None + + +class CollectAt(sqlmodel.SQLModel): + """Historical migration-only value retained for the append-only baseline.""" + + day_of_week: int | None = None + hour: int = 0 + minute: int = 0 + + +class CollectAtType(sqlmodel.TypeDecorator): + """Historical migration-only PostgreSQL type; not a current Source field.""" + + impl = sqlalchemy.dialects.postgresql.JSONB + cache_ok = True + + def process_bind_param(self, value, dialect): + del dialect + return value.model_dump() if isinstance(value, CollectAt) else value + + def process_result_value(self, value, dialect): + del dialect + return CollectAt.model_validate(value) if value else None + + +class SourceCollectParameters(pydantic.BaseModel): + """Parameters of the exact ordinary Source Job type.""" + + model_config = pydantic.ConfigDict(extra="forbid") + + source: SourceID + config: dict[str, typing.Any] = pydantic.Field(default_factory=dict) + + +class SourceBackfillParameters(SourceCollectParameters): + """Parameters of the exact historical Source Job type.""" + + +class SourceTypesModel(sqlmodel.SQLModel, table=True): + __tablename__: str = "sources_types" # type: ignore + + id: str = sqlmodel.Field(sa_column=sqlalchemy.Column(sqlalchemy.Text, primary_key=True)) + """Type of source. + + An absolute import path to the module where souce class at. + """ + description: str = sqlmodel.Field(sa_column=sqlalchemy.Column(sqlalchemy.Text)) + """Description of this source type.""" + config_schema: dict = sqlmodel.Field( + sa_column=sqlalchemy.Column( + sqlalchemy.dialects.postgresql.JSONB, + server_default=sqlalchemy.text("'{}'::jsonb"), + nullable=False, + ), + default=dict, + ) + collect_config_schema: dict = sqlmodel.Field( + sa_column=sqlalchemy.Column( + sqlalchemy.dialects.postgresql.JSONB, + server_default=sqlalchemy.text("'{}'::jsonb"), + nullable=False, + ), + default=dict, + ) + backfill_config_schema: dict | None = sqlmodel.Field( + sa_column=sqlalchemy.Column( + sqlalchemy.dialects.postgresql.JSONB, + nullable=True, + ), + default=None, + ) + + +class SourceModel(sqlmodel.SQLModel, table=True): + __tablename__: str = "sources" # type: ignore + + id: Opt[SourceID] = sqlmodel.Field( + sa_column=sqlalchemy.Column(sqlalchemy.Integer, primary_key=True, autoincrement=True), + default=None, + ) + type: str = sqlmodel.Field( + sa_column=sqlalchemy.Column( + sqlalchemy.Text, + sqlalchemy.ForeignKey("sources_types.id", onupdate="CASCADE", ondelete="CASCADE"), + ) + ) + """Type of source. + + An absolute import path to the module where souce class at. + When delete a source type, all sources of this type will be deleted too. + """ + nickname: Opt[str] = sqlmodel.Field( + default=None, + sa_column=sqlalchemy.Column(sqlalchemy.Text, nullable=True), + ) + config: dict = sqlmodel.Field( + sa_column=sqlalchemy.Column( + sqlalchemy.dialects.postgresql.JSONB, + server_default=sqlalchemy.text("'{}'::jsonb"), + nullable=False, + ), + default=dict, + ) + storage: Opt[StorageID] = sqlmodel.Field( + default=None, + sa_column=sqlalchemy.Column( + sqlalchemy.Integer, + sqlalchemy.ForeignKey("storages.id", onupdate="CASCADE", ondelete="RESTRICT"), + nullable=True, + ), + ) + block: Opt[BlockID] = sqlmodel.Field( + default=None, + sa_column=sqlalchemy.Column( + sqlalchemy.Integer, + sqlalchemy.ForeignKey("blocks.id", onupdate="CASCADE", ondelete="SET NULL"), + nullable=True, + unique=True, + ), + ) + state: dict = sqlmodel.Field( + sa_column=sqlalchemy.Column( + sqlalchemy.dialects.postgresql.JSONB, + server_default=sqlalchemy.text("'{}'::jsonb"), + nullable=False, + ), + default_factory=dict, + ) + """Store source-specific state (e.g., last_uid for mail, + latest_tweet_id for twitter).""" + created_at: datetime.datetime = sqlmodel.Field( + default=None, + sa_column=sqlalchemy.Column( + sqlalchemy.TIMESTAMP(timezone=True), + nullable=False, + server_default=sqlalchemy.text("CURRENT_TIMESTAMP"), + ), + ) + updated_at: datetime.datetime = sqlmodel.Field( + default=None, + sa_column=sqlalchemy.Column( + sqlalchemy.TIMESTAMP(timezone=True), + nullable=False, + server_default=sqlalchemy.text("CURRENT_TIMESTAMP"), + ), + ) diff --git a/utils/AGENTS.md b/inkcre_core/utils/AGENTS.md similarity index 100% rename from utils/AGENTS.md rename to inkcre_core/utils/AGENTS.md diff --git a/inkcre_core/utils/__init__.py b/inkcre_core/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/inkcre_core/utils/base.py b/inkcre_core/utils/base.py new file mode 100644 index 00000000..f0fd8aae --- /dev/null +++ b/inkcre_core/utils/base.py @@ -0,0 +1,15 @@ +import enum +import aiohttp +import ssl +import certifi +import pydantic + + +enum_serializer = pydantic.PlainSerializer( + lambda value: value.value if isinstance(value, enum.Enum) else value, + return_type=str, +) + + +def AIOHTTP_CONNECTOR_GETTER(): + return aiohttp.TCPConnector(ssl=ssl.create_default_context(cafile=certifi.where())) diff --git a/inkcre_core/utils/datetime_.py b/inkcre_core/utils/datetime_.py new file mode 100644 index 00000000..a063f0ce --- /dev/null +++ b/inkcre_core/utils/datetime_.py @@ -0,0 +1,96 @@ +""" +author: Lan_zhijiang +date: 2024-11-10 +desc: 时间日期工具函数 +issues: + +references: + https://git.hadream.ltd/anana/backend/common/-/wikis/Util/Datetime +""" + +import datetime +import time + +timezone_delta: int = 8 # 默认东八区 + + +def get_timezone(timezone_delta: int = timezone_delta) -> datetime.timezone: + """ + 获取时区对象 + """ + return datetime.timezone(datetime.timedelta(hours=timezone_delta)) + + +def get_datetimez( + rfc3339: str | None = None, + timestamp: float | None = None, + iso8601: str | None = None, + timezone: datetime.timezone = get_timezone(), +) -> datetime.datetime: + """ + 获取当前时间对象（带时区） + + :param rfc3339: RFC3339格式时间字符串，如"2024-11-10T12:00:00+08:00" + :param timestamp: 时间戳（秒级浮点）；如果使用此，时区固定为 UTC+0 + :param iso8601: ISO8601格式时间字符串，如"2024-11-10 12:00:00" + :param timezone: 时区对象，可以通过 `get_timezone(8)` 获取东八区时区 + :raise ValueError: 如果rfc3339或iso8601都未提供，则抛出异常 + """ + if timestamp: + return datetime.datetime.fromtimestamp(timestamp, tz=get_timezone(0)) + if rfc3339 or iso8601: + val = rfc3339 or iso8601 + if val is None: + raise ValueError("rfc3339 or iso8601 must be provided") + return datetime.datetime.fromisoformat(val) + return datetime.datetime.now(tz=timezone) + + +def get_datetime(timestamp: float | None = None) -> datetime.datetime: + """ + 获取当前时间对象（不带时区） + """ + if timestamp: + return datetime.datetime.fromtimestamp(timestamp) + return datetime.datetime.now() + + +def get_timestamp( + datetime_obj: datetime.datetime | None = None, rfc3339: str | None = None +) -> float: + """ + 获取时间戳（秒级浮点） + """ + if datetime_obj: + return datetime_obj.timestamp() + if rfc3339: + return datetime.datetime.fromisoformat(rfc3339).timestamp() + return time.time() + + +def get_rfc3339( + timestamp: float | None = None, datetime_obj: datetime.datetime | None = None +) -> str: + """ + 获取RFC3339格式时间 + """ + if timestamp: + return datetime.datetime.fromtimestamp(timestamp).isoformat() + elif datetime_obj: + return datetime_obj.isoformat() + + raise ValueError("timestamp or datetime_obj must be provided") + + +def format_datetime(datetime: datetime.datetime, format: str = "%Y-%m-%d %H:%M:%S") -> str: + """ + 格式化时间对象 + """ + return datetime.strftime(format) + + +def format_timestamp(timestamp: float, format: str = "%Y-%m-%d %H:%M:%S") -> str: + """ + 格式化时间戳 + """ + return datetime.datetime.fromtimestamp(timestamp).strftime(format) diff --git a/inkcre_core/utils/sql.py b/inkcre_core/utils/sql.py new file mode 100644 index 00000000..6670ac97 --- /dev/null +++ b/inkcre_core/utils/sql.py @@ -0,0 +1,32 @@ +"""Sql utilities.""" + +import typing +import sqlalchemy +from sqlalchemy.dialects.postgresql import JSONB + + +def find_by_json_field(column: typing.Any, field_name: str, value: str): + """Find by JSON field value. + + :param column: sqlalchemy column + + Examples: + >>> find_by_json_field(YourTable.column_name, "email", "example@example.com") + """ + return sqlalchemy.cast(column, JSONB)[field_name].astext == value + + +def find_by_nested_json(column: typing.Any, parent: str, child: str, value: str): + """Find by nested JSON field value. + + :param column: sqlalchemy column + """ + return sqlalchemy.cast(column, JSONB)[(parent, child)].astext == value + + +def find_by_json_contains(column: typing.Any, json_obj: dict): + """Find by JSON contains. + + :param column: sqlalchemy column + """ + return sqlalchemy.cast(column, JSONB).contains(json_obj) diff --git a/inkcre_core/utils/types_.py b/inkcre_core/utils/types_.py new file mode 100644 index 00000000..60f22cd0 --- /dev/null +++ b/inkcre_core/utils/types_.py @@ -0,0 +1,10 @@ +__all__ = ["Undefined", "_undefined"] + +import enum + + +class Undefined(enum.Enum): + _undefined = 0 + + +_undefined = Undefined._undefined diff --git a/inkcre_core/validation.py b/inkcre_core/validation.py new file mode 100644 index 00000000..beaee8e2 --- /dev/null +++ b/inkcre_core/validation.py @@ -0,0 +1,31 @@ +"""Path projection for nested input contracts; validation stays with their owners.""" + +import contextlib +import typing + +import jsonschema # pyrefly: ignore[untyped-import] +import pydantic + + +@contextlib.contextmanager +def input_path(*prefix: str | int): + """Attach a form nesting path to native errors without validating again.""" + try: + yield + except pydantic.ValidationError as error: + details = [ + {**item, "loc": (*prefix, *item["loc"])} for item in error.errors(include_url=False) + ] + raise pydantic.ValidationError.from_exception_data( + error.title, typing.cast(typing.Any, details) + ) from error + except jsonschema.ValidationError as error: + raise jsonschema.ValidationError( + error.message, + validator=error.validator, + validator_value=error.validator_value, + instance=error.instance, + schema=error.schema, + path=(*prefix, *error.absolute_path), + schema_path=error.absolute_schema_path, + ) from error diff --git a/libs/obsrv/__init__.py b/libs/obsrv/__init__.py index 39a4f499..43d3a30d 100644 --- a/libs/obsrv/__init__.py +++ b/libs/obsrv/__init__.py @@ -1,5 +1,3 @@ -"""Observability package.""" +"""Legacy import; use inkcre_core for new consumers.""" -__all__ = ["get_logger", "setup_obsrv"] - -from .main import get_logger, setup_obsrv +from inkcre_core.libs.obsrv import * diff --git a/libs/obsrv/log_handler_logtail.py b/libs/obsrv/log_handler_logtail.py index 61a93592..8e081e1c 100644 --- a/libs/obsrv/log_handler_logtail.py +++ b/libs/obsrv/log_handler_logtail.py @@ -1,29 +1,10 @@ -"""Logtail logging handler.""" +"""Legacy import; use inkcre_core for new consumers.""" -import logging -from typing import Optional +from typing import TYPE_CHECKING +from importlib import import_module +import sys -from logtail import LogtailHandler as RealLogtailHandler +sys.modules[__name__] = import_module("inkcre_core.libs.obsrv.log_handler_logtail") - -class LogtailHandler: - """Wrapper for Logtail handler.""" - - def __init__(self, source_token: Optional[str] = None, host: Optional[str] = None): - self.source_token = source_token - self.host = host - self.handler = None - - def get_handler(self) -> Optional[logging.Handler]: - """Get the logtail handler if available.""" - if not self.source_token: - return None - if self.handler is None: - try: - self.handler = RealLogtailHandler(source_token=self.source_token) - if self.host: - self.handler.host = self.host - self.handler.setLevel(logging.INFO) - except Exception: - return None - return self.handler +if TYPE_CHECKING: + from inkcre_core.libs.obsrv.log_handler_logtail import * diff --git a/libs/obsrv/log_handler_postgresql.py b/libs/obsrv/log_handler_postgresql.py index ff4bbadd..0e4ebaa3 100644 --- a/libs/obsrv/log_handler_postgresql.py +++ b/libs/obsrv/log_handler_postgresql.py @@ -1,75 +1,10 @@ -"""Bounded bridge from synchronous logging calls to async PostgreSQL writes.""" +"""Legacy import; use inkcre_core for new consumers.""" -import asyncio -import logging -from queue import Empty, Full, Queue +from typing import TYPE_CHECKING +from importlib import import_module import sys -from libs.obsrv.log_record import ENABLE_LOG_BACKEND, LogModel, adapt_log_record +sys.modules[__name__] = import_module("inkcre_core.libs.obsrv.log_handler_postgresql") - -class PostgreSQLHandler(logging.Handler): - """Capture caller context immediately; persist only while lifespan is running.""" - - def __init__(self, level: int): - super().__init__(level) - self._queue: Queue[LogModel] = Queue(maxsize=1024) - self._task: asyncio.Task[None] | None = None - self._accepting = False - self.dropped = 0 - - def start(self) -> None: - if self._task is not None: - raise RuntimeError("PostgreSQL log writer is already running") - self._accepting = True - self._task = asyncio.create_task(self._write(), name="postgresql-log-writer") - - def emit(self, record: logging.LogRecord) -> None: - if not self._accepting or not ENABLE_LOG_BACKEND.get(): - return - try: - self._queue.put_nowait(adapt_log_record(record)) - except Full: - self.dropped += 1 - if self.dropped == 1: - print("PostgreSQL log queue full; dropping backend records", file=sys.stderr) - except Exception: - self.handleError(record) - - async def _write(self) -> None: - from app.persistence.observability.uow import write_logs - - while self._accepting or not self._queue.empty(): - batch: list[LogModel] = [] - for _ in range(100): - try: - batch.append(self._queue.get_nowait()) - except Empty: - break - if not batch: - await asyncio.sleep(0.1) - continue - try: - await write_logs(batch) - except Exception as error: - # Report through stderr, never recursively through the failed backend. - print(f"PostgreSQL log batch failed: {type(error).__name__}", file=sys.stderr) - - async def aclose(self) -> None: - # Handler.handle() holds this lock even when emit originates on a worker thread. - self.acquire() - try: - self._accepting = False - finally: - self.release() - if self._task is not None: - try: - async with asyncio.timeout(5): - await self._task - except TimeoutError: - print("PostgreSQL log drain timed out; pending records lost", file=sys.stderr) - finally: - self._task = None - while not self._queue.empty(): - self._queue.get_nowait() - super().close() +if TYPE_CHECKING: + from inkcre_core.libs.obsrv.log_handler_postgresql import * diff --git a/libs/obsrv/log_record.py b/libs/obsrv/log_record.py index 16531203..c063876c 100644 --- a/libs/obsrv/log_record.py +++ b/libs/obsrv/log_record.py @@ -1,119 +1,10 @@ -__all__ = ["LogModel", "adapt_log_record", "TRACE_ID", "SPAN_ID", "ENABLE_LOG_BACKEND"] +"""Legacy import; use inkcre_core for new consumers.""" -from contextvars import ContextVar -import logging -import datetime -from typing import Mapping, Any, Dict, Optional as Opt -import sqlalchemy as sa -from sqlalchemy.dialects.postgresql import JSONB -import sqlmodel +from typing import TYPE_CHECKING +from importlib import import_module +import sys -from app.database_contract.constants import PROTOCOL_SCHEMA +sys.modules[__name__] = import_module("inkcre_core.libs.obsrv.log_record") - -TRACE_ID: ContextVar[str | None] = ContextVar("TRACE_ID", default=None) -SPAN_ID: ContextVar[str | None] = ContextVar("SPAN_ID", default=None) -ENABLE_LOG_BACKEND: ContextVar[bool] = ContextVar("ENABLE_LOG_BACKEND", default=False) -"""Whether the log backend emit the logs or not""" - - -LOGGING_TO_OTEL: Mapping[int, int] = { - 10: 5, # DEBUG - 20: 9, # INFO - 30: 13, # WARNING - 40: 17, # ERROR - 50: 21, # CRITICAL -} - - -def logging_level_to_otel(levelno: int) -> int: - return LOGGING_TO_OTEL.get(levelno, 9) - - -class LogModel(sqlmodel.SQLModel, table=True): - """Log model""" - - __tablename__ = "logs" # type: ignore - __table_args__ = {"schema": PROTOCOL_SCHEMA} - - id: Opt[int] = sqlmodel.Field( - default=None, - sa_column=sa.Column(sa.BigInteger, primary_key=True, autoincrement=True), - ) - timestamp: datetime.datetime = sqlmodel.Field( - default_factory=datetime.datetime.now, - sa_column=sa.Column( - sa.TIMESTAMP(timezone=True), - server_default=sa.text("CURRENT_TIMESTAMP"), - nullable=False, - ), - ) - severity_number: int = sqlmodel.Field(sa_type=sa.SmallInteger) - severity_text: str = sqlmodel.Field(sa_column=sa.Column(sa.Text, nullable=False)) - body: str = sqlmodel.Field(sa_column=sa.Column(sa.Text, nullable=False)) - trace_id: Opt[str] = sqlmodel.Field( - default=None, - sa_column=sa.Column(sa.Text, nullable=True), - ) - span_id: Opt[str] = sqlmodel.Field( - default=None, - sa_column=sa.Column(sa.Text, nullable=True), - ) - attributes: Dict[str, Any] = sqlmodel.Field( - sa_type=JSONB, - default_factory=dict, - sa_column_kwargs={"server_default": sa.text("'{}'::jsonb")}, - ) - - -def adapt_log_record( - record: logging.LogRecord, - # *, - # service_name: str, - # service_version: str | None = None, - # environment: str | None = None, -) -> LogModel: - attributes = { - k: v - for k, v in record.__dict__.items() - if k - not in { - "name", - "msg", - "message", - "args", - "levelname", - "levelno", - "pathname", - "filename", - "module", - "exc_info", - "exc_text", - "stack_info", - "lineno", - "funcName", - "created", - "msecs", - "relativeCreated", - "thread", - "threadName", - "processName", - "process", - "taskName", - "asctime", - } - and not k.startswith("_") - } - - return LogModel( - timestamp=datetime.datetime.fromtimestamp(record.created), - severity_number=logging_level_to_otel(record.levelno), - severity_text=record.levelname, - body=record.getMessage(), - trace_id=TRACE_ID.get(), - span_id=SPAN_ID.get(), - attributes=attributes, - # service_name=service_name, - # service_version=service_version, - # environment=environment, - ) +if TYPE_CHECKING: + from inkcre_core.libs.obsrv.log_record import * diff --git a/libs/obsrv/main.py b/libs/obsrv/main.py index 19e16ca2..c3674a5b 100644 --- a/libs/obsrv/main.py +++ b/libs/obsrv/main.py @@ -1,85 +1,10 @@ -"""Main observability setup.""" +"""Legacy import; use inkcre_core for new consumers.""" -import logging +from typing import TYPE_CHECKING +from importlib import import_module import sys -__all__ = ["get_logger"] +sys.modules[__name__] = import_module("inkcre_core.libs.obsrv.main") -LOGGER = logging.getLogger("inkcre") - - -def setup_obsrv() -> logging.Logger: - """Setup observability components.""" - from app.settings import settings - - # Create logger - global LOGGER - LOGGER = logging.getLogger("inkcre") - LOGGER.setLevel(logging.DEBUG) - - # Clear any existing handlers - LOGGER.handlers.clear() - - # Console handler (always enabled) - console_handler = logging.StreamHandler(sys.stdout) - console_handler.setLevel(logging.INFO) - console_formatter = logging.Formatter( - "%(asctime)s - %(name)s - %(levelname)s - %(message)s" - ) - console_handler.setFormatter(console_formatter) - LOGGER.addHandler(console_handler) - - # Add backend-specific handlers - backend = settings.obsrv.logging_backend - if backend == "logtail": - from .log_handler_logtail import LogtailHandler - - logtail_wrapper = LogtailHandler( - source_token=settings.obsrv.logtail_source_token, - host=settings.obsrv.logtail_host, - ) - handler = logtail_wrapper.get_handler() - if handler: - LOGGER.addHandler(handler) - LOGGER.info("Logtail logging enabled") - else: - LOGGER.warning("Logtail logging configured but not available") - elif backend == "postgresql": - from .log_handler_postgresql import PostgreSQLHandler - - pg_handler = PostgreSQLHandler(level=settings.obsrv.logging_backend_level) - LOGGER.addHandler(pg_handler) - LOGGER.info("PostgreSQL logging enabled") - elif backend not in (None, "", "none"): - LOGGER.warning(f"Unknown logging backend: {backend}") - - return LOGGER - - -def get_logger() -> logging.Logger: - """Get the application logger instance. - - Returns: - Logger instance for the application - """ - global LOGGER - return LOGGER - - -def start_obsrv() -> None: - from .log_handler_postgresql import PostgreSQLHandler - - for handler in LOGGER.handlers: - if isinstance(handler, PostgreSQLHandler): - handler.start() - - -async def close_obsrv() -> None: - from .log_handler_postgresql import PostgreSQLHandler - from .telemetry import close_telemetry - - await close_telemetry() - - for handler in LOGGER.handlers: - if isinstance(handler, PostgreSQLHandler): - await handler.aclose() +if TYPE_CHECKING: + from inkcre_core.libs.obsrv.main import * diff --git a/libs/obsrv/setting.py b/libs/obsrv/setting.py index 8e3a3346..aad5c4b7 100644 --- a/libs/obsrv/setting.py +++ b/libs/obsrv/setting.py @@ -1,35 +1,10 @@ -"""Observability settings.""" +"""Legacy import; use inkcre_core for new consumers.""" -from typing import Optional +from typing import TYPE_CHECKING +from importlib import import_module +import sys -from pydantic import Field -from pydantic_settings import BaseSettings +sys.modules[__name__] = import_module("inkcre_core.libs.obsrv.setting") - -class ObsrvSetting(BaseSettings): - """Observability settings.""" - - telemetry_enabled: bool = Field( - default=False, - description="Enable optional metadata-only OpenTelemetry export for this Peer.", - ) - - agent_debug: bool = Field( - default=False, - description="Record Agent inputs, tool contracts and execution events for development.", - ) - - logging_backend: Optional[str] = Field( - default="postgresql", - description="Logging backend to use (e.g., 'logtail', 'postgresql')", - ) - logging_backend_level: int = Field( - default=20, - description="Minimum log level for sending logs to the backend", - ) - logtail_source_token: Optional[str] = Field( - default=None, description="Better Stack (Logtail) source token for remote logging" - ) - logtail_host: Optional[str] = Field( - default=None, description="Better Stack (Logtail) host URL" - ) +if TYPE_CHECKING: + from inkcre_core.libs.obsrv.setting import * diff --git a/libs/obsrv/telemetry.py b/libs/obsrv/telemetry.py index 6343e9d9..1aa47f4f 100644 --- a/libs/obsrv/telemetry.py +++ b/libs/obsrv/telemetry.py @@ -1,488 +1,10 @@ -"""Opt-in OTLP metadata transport; callers own allowed field names and value sources.""" +"""Legacy import; use inkcre_core for new consumers.""" -import asyncio -from collections.abc import Callable, Generator, Mapping, Sequence -from contextlib import ExitStack, contextmanager -from dataclasses import dataclass, field -from functools import partial -import logging -import math -import os +from typing import TYPE_CHECKING +from importlib import import_module import sys -import time -from typing import TYPE_CHECKING, Literal -from urllib.parse import urlsplit -import uuid -from opentelemetry.context import Context -from opentelemetry.trace import INVALID_SPAN, Link, Span, SpanKind, StatusCode, use_span -from opentelemetry.util.types import AttributeValue -from pydantic import SecretStr -from pydantic_settings import BaseSettings, SettingsConfigDict +sys.modules[__name__] = import_module("inkcre_core.libs.obsrv.telemetry") if TYPE_CHECKING: - from opentelemetry.sdk._logs import LoggerProvider - from opentelemetry.sdk.metrics import MeterProvider - from opentelemetry.sdk.trace import TracerProvider - from opentelemetry.metrics import Counter, Histogram - from opentelemetry.trace import Tracer - from opentelemetry._logs import Logger - - -class _ExportSettings(BaseSettings): - model_config = SettingsConfigDict(extra="ignore") - - otel_exporter_otlp_traces_endpoint: str = "" - otel_exporter_otlp_logs_endpoint: str = "" - otel_exporter_otlp_metrics_endpoint: str = "" - otel_exporter_otlp_headers: SecretStr = SecretStr("") - otel_exporter_otlp_traces_headers: SecretStr = SecretStr("") - otel_exporter_otlp_logs_headers: SecretStr = SecretStr("") - otel_exporter_otlp_metrics_headers: SecretStr = SecretStr("") - # Validate each signal separately, so a bad timeout does not disable siblings. - otel_exporter_otlp_timeout: str = "10" - otel_exporter_otlp_traces_timeout: str = "" - otel_exporter_otlp_logs_timeout: str = "" - otel_exporter_otlp_metrics_timeout: str = "" - - -@dataclass -class _Runtime: - connections: dict[str, tuple[str, dict[str, str], float]] = field(default_factory=dict) - traces: "TracerProvider | None" = None - logs: "LoggerProvider | None" = None - metrics: "MeterProvider | None" = None - tracer: "Tracer | None" = None - logger: "Logger | None" = None - operations: "Counter | None" = None - duration: "Histogram | None" = None - tokens: "Counter | None" = None - missing_usage: "Counter | None" = None - - -_runtime: _Runtime | None = None - - -def _diagnostic(reason: str) -> None: - # This sink never receives endpoints, headers, exceptions, or arbitrary log messages. - try: - print(f"OpenTelemetry: {reason}", file=sys.stderr) - except Exception: - # Diagnostics cannot make a broken stderr stream a business failure. - pass - - -class _SDKDiagnostics(logging.Handler): - def emit(self, record: logging.LogRecord) -> None: - # SDK errors may contain response bodies, URLs or malformed secret headers. - _diagnostic("SDK warning or export failure; check the configured OTLP receiver") - - -def _configure_diagnostics() -> None: - logger = logging.getLogger("opentelemetry") - if not any(isinstance(handler, _SDKDiagnostics) for handler in logger.handlers): - handler = _SDKDiagnostics(level=logging.WARNING) - logger.addHandler(handler) - logger.propagate = False - - -def _endpoint(value: str) -> str: - parsed = urlsplit(value) - if ( - parsed.scheme not in ("http", "https") - or not parsed.hostname - or not parsed.path - or parsed.username is not None - or parsed.password is not None - or parsed.query - or parsed.fragment - or any(character.isspace() for character in value) - ): - raise ValueError("invalid OTLP URL") - # Accessing port performs urllib's range and syntax validation. - parsed.port - return value - - -def is_enabled() -> bool: - """Whether at least one metadata signal initialized for this runtime.""" - return _runtime is not None - - -def get_export_connection(signal: str) -> tuple[str, dict[str, str], float] | None: - """Return private endpoint, copied headers and timeout seconds for server forwarding.""" - connection = _runtime.connections.get(signal) if _runtime is not None else None - return ( - (connection[0], dict(connection[1]), connection[2]) if connection is not None else None - ) - - -def start_telemetry( - *, - enabled: bool, - service_version: str, - resource_attributes: Mapping[str, AttributeValue] | None = None, - endpoints: Mapping[str, str] | None = None, -) -> None: - """Initialize only configured signals; configuration failures never gate readiness.""" - global _runtime - if not enabled or _runtime is not None: - return - - _configure_diagnostics() - try: - config = _ExportSettings(_env_file=os.getenv("INKCRE_ENV_FILE", ".env") or None) - except Exception: - _diagnostic("invalid runtime export configuration; telemetry is disabled") - return - - from opentelemetry.exporter.otlp.proto.http import Compression - from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter - from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter - from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter - from opentelemetry.metrics import NoOpMeterProvider - from opentelemetry.sdk._logs import LoggerProvider, LogRecordLimits - from opentelemetry.sdk._logs.export import BatchLogRecordProcessor - from opentelemetry.sdk.metrics import MeterProvider, SimpleFixedSizeExemplarReservoir - from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader - from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View - from opentelemetry.sdk.resources import Resource - from opentelemetry.sdk.trace import SpanLimits, TracerProvider - from opentelemetry.sdk.trace.export import BatchSpanProcessor - from opentelemetry.sdk.trace.sampling import ALWAYS_ON - from opentelemetry.util.re import parse_env_headers - from requests.utils import check_header_validity # type: ignore[untyped-import] - - # Resource.create invokes environment detectors; direct construction is deliberate. - resource = Resource( - { - **(resource_attributes or {}), - "service.name": "core-py", - "service.version": service_version, - "service.instance.id": str(uuid.uuid4()), - } - ) - runtime = _Runtime() - no_metrics = NoOpMeterProvider() - for signal in ("metrics", "traces", "logs"): - endpoint = getattr(config, f"otel_exporter_otlp_{signal}_endpoint") or ( - endpoints or {} - ).get(signal, "") - if not endpoint: - continue - cleanup: Callable[[], None] | None = None - internal_metrics = runtime.metrics or no_metrics - try: - endpoint = _endpoint(endpoint) - timeout = float( - getattr(config, f"otel_exporter_otlp_{signal}_timeout") - or config.otel_exporter_otlp_timeout - or "10" - ) - if not math.isfinite(timeout) or not 0 < timeout <= 30: - raise ValueError("OTLP timeout must be finite, positive and at most 30 seconds") - raw_headers = ( - getattr(config, f"otel_exporter_otlp_{signal}_headers").get_secret_value() - or config.otel_exporter_otlp_headers.get_secret_value() - ) - headers = parse_env_headers(raw_headers, liberal=True) - # The SDK parser drops malformed entries. Do not silently accept partial auth. - if len(headers) != len([part for part in raw_headers.split(",") if part.strip()]): - raise ValueError("invalid OTLP headers") - for header in headers.items(): - check_header_validity(header) - - if signal == "traces": - exporter = OTLPSpanExporter( - endpoint=endpoint, - headers=headers, - timeout=timeout, - compression=Compression.NoCompression, - meter_provider=internal_metrics, - ) - cleanup = exporter.shutdown - traces = TracerProvider( - resource=resource, - sampler=ALWAYS_ON, - shutdown_on_exit=False, - span_limits=SpanLimits( - max_attributes=64, - max_events=16, - max_links=16, - max_attribute_length=256, - max_span_attributes=64, - max_event_attributes=64, - max_link_attributes=64, - max_span_attribute_length=256, - ), - meter_provider=internal_metrics, - ) - processor = BatchSpanProcessor( - exporter, - max_queue_size=256, - max_export_batch_size=256, - schedule_delay_millis=5000, - export_timeout_millis=timeout * 1000, - meter_provider=internal_metrics, - ) - cleanup = processor.shutdown - traces.add_span_processor(processor) - cleanup = traces.shutdown - tracer = traces.get_tracer("inkcre.metadata") - runtime.traces, runtime.tracer = traces, tracer - elif signal == "logs": - log_exporter = OTLPLogExporter( - endpoint=endpoint, - headers=headers, - timeout=timeout, - compression=Compression.NoCompression, - meter_provider=internal_metrics, - ) - cleanup = log_exporter.shutdown - logs = LoggerProvider( - resource=resource, - shutdown_on_exit=False, - meter_provider=internal_metrics, - log_record_limits=LogRecordLimits(max_attributes=64, max_attribute_length=256), - ) - log_processor = BatchLogRecordProcessor( - log_exporter, - max_queue_size=256, - max_export_batch_size=256, - schedule_delay_millis=5000, - export_timeout_millis=timeout * 1000, - meter_provider=internal_metrics, - ) - cleanup = log_processor.shutdown - logs.add_log_record_processor(log_processor) - cleanup = logs.shutdown - logger = logs.get_logger("inkcre.metadata") - runtime.logs, runtime.logger = logs, logger - else: - metric_exporter = OTLPMetricExporter( - endpoint=endpoint, - headers=headers, - timeout=timeout, - compression=Compression.NoCompression, - meter_provider=no_metrics, - ) - cleanup = metric_exporter.shutdown - reader = PeriodicExportingMetricReader( - metric_exporter, - export_interval_millis=30000, - export_timeout_millis=timeout * 1000, - ) - cleanup = partial(reader.shutdown, timeout_millis=(timeout + 2) * 1000) - metrics = MeterProvider( - resource=resource, - metric_readers=[reader], - shutdown_on_exit=False, - views=[ - # SDK defaults are too coarse for a duration measured in seconds. - View( - instrument_name="inkcre.operation.duration", - aggregation=ExplicitBucketHistogramAggregation( - boundaries=( - 0.005, - 0.01, - 0.025, - 0.05, - 0.1, - 0.25, - 0.5, - 1, - 2.5, - 5, - 10, - 30, - 60, - 300, - ) - ), - ), - View( - meter_name="opentelemetry-sdk", - attribute_keys={ - "otel.component.type", - "error.type", - "http.response.status_code", - }, - # An exemplar can otherwise retain attributes excluded by the View. - exemplar_reservoir_factory=lambda _: ( - lambda _config=None, **_kwargs: SimpleFixedSizeExemplarReservoir(size=0) - ), - ), - ], - ) - cleanup = partial(metrics.shutdown, timeout_millis=(timeout + 2) * 1000) - meter = metrics.get_meter("inkcre.metadata") - operations = meter.create_counter("inkcre.operation.count", unit="{operation}") - duration = meter.create_histogram("inkcre.operation.duration", unit="s") - tokens = meter.create_counter("inkcre.ai.token.usage", unit="{token}") - missing_usage = meter.create_counter("inkcre.ai.usage.missing", unit="{field}") - runtime.metrics = metrics - runtime.operations, runtime.duration = operations, duration - runtime.tokens, runtime.missing_usage = tokens, missing_usage - except Exception: - if cleanup is not None: - _shutdown(cleanup) - _diagnostic(f"{signal} configuration or initialization failed; signal is disabled") - else: - runtime.connections[signal] = (endpoint, dict(headers), timeout) - - if any((runtime.traces, runtime.logs, runtime.metrics)): - _runtime = runtime - else: - _diagnostic("no usable per-signal endpoint; telemetry is disabled") - - -def _shutdown(shutdown: Callable[[], None]) -> None: - try: - shutdown() - except Exception: - _diagnostic("signal shutdown failed; remaining telemetry may be lost") - - -async def close_telemetry() -> None: - """Stop new records and concurrently drain SDK queues outside the event loop. - - SDK trace/log shutdown waits up to 30 seconds for each worker, then metrics gets - its configured request timeout plus two seconds. These worker wait budgets are - not transport-wide hard deadlines; freezing or termination can lose records. - """ - global _runtime - runtime, _runtime = _runtime, None - if runtime is None: - return - shutdowns: list[Callable[[], None]] = [] - for provider in (runtime.traces, runtime.logs): - if provider is not None: - shutdowns.append(provider.shutdown) - # SDK 1.45 force_flush ignores its timeout; shutdown owns the bounded queue drain. - await asyncio.gather(*(asyncio.to_thread(_shutdown, shutdown) for shutdown in shutdowns)) - if runtime.metrics is not None: - # Collect final native processor/exporter metrics after their workers have drained. - await asyncio.to_thread( - _shutdown, - partial( - runtime.metrics.shutdown, - timeout_millis=(runtime.connections["metrics"][2] + 2) * 1000, - ), - ) - - -def _record(action: Callable[[], object]) -> None: - """Isolate one owned telemetry call; never wrap the caller's business block.""" - try: - action() - except Exception: - _diagnostic("recording failed; optional telemetry may be lost") - - -@dataclass -class Operation: - """One execution scope's result, independent of whether its Span is recorded. - - Business boundaries set outcome when they return a failure or handle cancellation. - Span methods remain available explicitly through span; Span status is not a metric. - Those standard SDK methods require the caller's admitted primitive metadata values; - the facade isolates its own SDK calls, not arbitrary caller code inside the scope. - """ - - span: Span - outcome: Literal["success", "error", "cancelled"] = "success" - - -@contextmanager -def operation( - name: str, - *, - attributes: Mapping[str, AttributeValue] | None = None, - context: Context | None = None, - links: Sequence[Link] | None = None, - kind: SpanKind = SpanKind.INTERNAL, -) -> Generator[Operation, None, None]: - """Measure a fixed operation name; callers supply only admitted metadata values. - - Exceptions and cancellation propagate unchanged. Their messages, stack traces and - status descriptions are never recorded by this context manager. - """ - runtime = _runtime - if runtime is None: - yield Operation(INVALID_SPAN) - return - - span = INVALID_SPAN - if runtime.tracer is not None: - try: - span = runtime.tracer.start_span( - name, - attributes=attributes, - context=context, - links=links, - kind=kind, - record_exception=False, - set_status_on_exception=False, - ) - except Exception: - _diagnostic("span creation failed; operation metrics remain available") - observation = Operation(span) - started = time.perf_counter() - activation = ExitStack() - _record( - lambda: activation.enter_context( - use_span(span, record_exception=False, set_status_on_exception=False) - ) - ) - try: - yield observation - except BaseException as error: - observation.outcome = ( - "cancelled" if isinstance(error, asyncio.CancelledError) else "error" - ) - _record(partial(span.set_attribute, "error.type", type(error).__name__)) - raise - finally: - # Exit the SDK context with no exception and end separately. This prevents SDK - # context cleanup or span processors from masking the caller's original error. - if observation.outcome != "success": - _record(partial(span.set_status, StatusCode.ERROR)) - _record(partial(span.set_attribute, "inkcre.outcome", observation.outcome)) - labels = {"operation": name, "outcome": observation.outcome} - if runtime.operations is not None: - _record(partial(runtime.operations.add, 1, labels)) - if runtime.duration is not None: - _record(partial(runtime.duration.record, time.perf_counter() - started, labels)) - _record(activation.close) - _record(span.end) - - -def emit_event(name: str, attributes: Mapping[str, AttributeValue]) -> None: - """Emit a fixed event name and admitted metadata without a Python logging bridge.""" - runtime = _runtime - if runtime is not None and runtime.logger is not None: - from opentelemetry._logs import SeverityNumber - - _record( - partial( - runtime.logger.emit, - body=name, - severity_number=SeverityNumber.INFO, - attributes=attributes, - ) - ) - - -def record_ai_usage( - operation: str, input_tokens: int | None, output_tokens: int | None -) -> None: - """Count known nonnegative provider usage and count missing fields separately.""" - runtime = _runtime - if runtime is None or runtime.tokens is None or runtime.missing_usage is None: - return - for kind, value in (("input", input_tokens), ("output", output_tokens)): - labels = {"operation": operation, "token.type": kind} - if isinstance(value, int) and not isinstance(value, bool) and value >= 0: - _record(partial(runtime.tokens.add, value, labels)) - else: - _record(partial(runtime.missing_usage.add, 1, labels)) + from inkcre_core.libs.obsrv.telemetry import * diff --git a/migrations/metadata.py b/migrations/metadata.py index ba93fd03..dcb08351 100644 --- a/migrations/metadata.py +++ b/migrations/metadata.py @@ -2,9 +2,9 @@ from sqlalchemy import MetaData -from app.database_contract import PROTOCOL_SCHEMA -from app.schemas import Base -from libs.obsrv.log_record import LogModel +from inkcre_core.database_contract import PROTOCOL_SCHEMA +from inkcre_core.schemas import Base +from inkcre_core.libs.obsrv.log_record import LogModel _qualified_log_table = f"{PROTOCOL_SCHEMA}.{LogModel.__tablename__}" diff --git a/pdm.lock b/pdm.lock index 7643e3fd..54f5c530 100644 --- a/pdm.lock +++ b/pdm.lock @@ -2,10 +2,10 @@ # It is not intended for manual editing. [metadata] -groups = ["default", "dev", "extension-preview", "extension-publisher"] +groups = ["default", "app", "dev", "extension-preview", "extension-publisher"] strategy = ["inherit_metadata"] lock_version = "4.5.0" -content_hash = "sha256:da973293e877e2840a1f344c4338b3fd1f808cb81bf02d5c80a9a3279585a924" +content_hash = "sha256:e8bc9b47fa7a1e15463f9582dbbbd990776ba38bb6a34ca59f0601ca130f17cf" [[metadata.targets]] requires_python = ">=3.12,<3.13" @@ -80,7 +80,7 @@ name = "alembic" version = "1.19.1" requires_python = ">=3.10" summary = "A database migration tool for SQLAlchemy." -groups = ["default"] +groups = ["app"] dependencies = [ "Mako", "SQLAlchemy>=1.4.23", @@ -97,7 +97,7 @@ name = "alembic-postgresql-enum" version = "1.10.0" requires_python = ">=3.7.0" summary = "Alembic autogenerate support for creation, alteration and deletion of enums" -groups = ["default"] +groups = ["app"] dependencies = [ "alembic>=1.7", "sqlalchemy>=1.4", @@ -112,7 +112,7 @@ name = "alembic-utils" version = "0.8.8" requires_python = ">=3.9" summary = "A sqlalchemy/alembic extension for migrating procedures and views" -groups = ["default"] +groups = ["app"] dependencies = [ "alembic>=1.9", "flupy", @@ -130,7 +130,7 @@ name = "annotated-doc" version = "0.0.5" requires_python = ">=3.9" summary = "Document parameters, class attributes, return types, and variables inline, with Annotated." -groups = ["default", "extension-preview", "extension-publisher"] +groups = ["app", "extension-preview", "extension-publisher"] files = [ {file = "annotated_doc-0.0.5-py3-none-any.whl", hash = "sha256:117bac03a25ede5df5440e855b32d556049ca169ead221505badf432fed4b101"}, {file = "annotated_doc-0.0.5.tar.gz", hash = "sha256:c7e58ce09192557605d8bbd92836d7e1d520ac9580096042c0bfd197efacf1bb"}, @@ -141,7 +141,7 @@ name = "annotated-types" version = "0.8.0" requires_python = ">=3.10" summary = "Reusable constraint types to use with typing.Annotated" -groups = ["default", "extension-preview", "extension-publisher"] +groups = ["default", "app", "extension-preview", "extension-publisher"] files = [ {file = "annotated_types-0.8.0-py3-none-any.whl", hash = "sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0"}, {file = "annotated_types-0.8.0.tar.gz", hash = "sha256:13b2beaad985e05e2d6407ee4c4f35590b11f8d693a258a561055cac8f64cab7"}, @@ -152,7 +152,7 @@ name = "anyio" version = "4.15.1" requires_python = ">=3.10" summary = "High-level concurrency and networking framework on top of asyncio or Trio" -groups = ["default", "extension-preview", "extension-publisher"] +groups = ["default", "app", "extension-preview", "extension-publisher"] dependencies = [ "exceptiongroup>=1.0.2; python_version < \"3.11\"", "idna>=2.8", @@ -168,7 +168,7 @@ name = "apscheduler" version = "3.11.3" requires_python = ">=3.8" summary = "In-process task scheduler with Cron-like capabilities" -groups = ["default"] +groups = ["app"] dependencies = [ "backports-zoneinfo; python_version < \"3.9\"", "tzlocal>=3.0", @@ -183,7 +183,7 @@ name = "attrs" version = "26.1.0" requires_python = ">=3.9" summary = "Classes Without Boilerplate" -groups = ["default"] +groups = ["default", "app"] files = [ {file = "attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309"}, {file = "attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32"}, @@ -194,7 +194,7 @@ name = "authlib" version = "1.7.2" requires_python = ">=3.10" summary = "The ultimate Python library in building OAuth and OpenID Connect servers and clients." -groups = ["default"] +groups = ["app"] dependencies = [ "cryptography", "joserfc>=1.6.0", @@ -228,7 +228,7 @@ name = "babel" version = "2.18.0" requires_python = ">=3.8" summary = "Internationalization utilities" -groups = ["default"] +groups = ["app"] dependencies = [ "pytz>=2015.7; python_version < \"3.9\"", ] @@ -242,7 +242,7 @@ name = "beautifulsoup4" version = "4.15.0" requires_python = ">=3.7.0" summary = "Screen-scraping library" -groups = ["default"] +groups = ["default", "app"] dependencies = [ "soupsieve>=1.6.1", "typing-extensions>=4.0.0", @@ -275,7 +275,7 @@ name = "certifi" version = "2026.7.22" requires_python = ">=3.7" summary = "Python package for providing Mozilla's CA Bundle." -groups = ["default", "extension-preview", "extension-publisher"] +groups = ["default", "app", "extension-preview", "extension-publisher"] files = [ {file = "certifi-2026.7.22-py3-none-any.whl", hash = "sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775"}, {file = "certifi-2026.7.22.tar.gz", hash = "sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55"}, @@ -286,7 +286,7 @@ name = "cffi" version = "2.1.1" requires_python = ">=3.10" summary = "Foreign Function Interface for Python calling C code." -groups = ["default"] +groups = ["app"] marker = "platform_python_implementation != \"PyPy\"" dependencies = [ "pycparser; implementation_name != \"PyPy\"", @@ -334,7 +334,7 @@ name = "charset-normalizer" version = "3.5.1" requires_python = ">=3.7" summary = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." -groups = ["default"] +groups = ["default", "app"] files = [ {file = "charset_normalizer-3.5.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:5b6d1386bf0096d26d3a863dc0a487a5b4eb9aa93cf5ba69683d29dde6b9d60f"}, {file = "charset_normalizer-3.5.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4582c27e8c889d64811987b5967fbd3ae0c823fe1fd933b543d55ac20bb475fa"}, @@ -377,7 +377,7 @@ name = "click" version = "8.4.2" requires_python = ">=3.10" summary = "Composable command line interface toolkit" -groups = ["default", "dev", "extension-preview", "extension-publisher"] +groups = ["app", "dev", "extension-preview", "extension-publisher"] dependencies = [ "colorama; platform_system == \"Windows\"", ] @@ -391,7 +391,7 @@ name = "colorama" version = "0.4.6" requires_python = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" summary = "Cross-platform colored terminal text." -groups = ["default", "dev", "extension-preview", "extension-publisher"] +groups = ["default", "app", "dev", "extension-preview", "extension-publisher"] marker = "os_name == \"nt\" or sys_platform == \"win32\" or platform_system == \"Windows\"" files = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, @@ -403,7 +403,7 @@ name = "courlan" version = "1.4.0" requires_python = ">=3.10" summary = "Clean, filter and sample URLs to optimize data collection – includes spam, content type and language filters." -groups = ["default"] +groups = ["app"] dependencies = [ "babel>=2.16.0", "tld>=0.13", @@ -433,7 +433,7 @@ name = "cryptography" version = "50.0.0" requires_python = ">=3.9, !=3.9.0, !=3.9.1" summary = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." -groups = ["default"] +groups = ["app"] dependencies = [ "cffi>=2.0.0; platform_python_implementation != \"PyPy\"", "typing-extensions>=4.13.2; python_full_version < \"3.11\"", @@ -495,7 +495,7 @@ name = "dateparser" version = "1.4.2" requires_python = ">=3.10" summary = "Date parsing library designed to parse dates from HTML pages" -groups = ["default"] +groups = ["app"] dependencies = [ "python-dateutil>=2.7.0", "pytz>=2024.2", @@ -512,7 +512,7 @@ name = "defusedxml" version = "0.7.1" requires_python = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" summary = "XML bomb protection for Python stdlib modules" -groups = ["default"] +groups = ["app"] files = [ {file = "defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"}, {file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"}, @@ -544,7 +544,7 @@ name = "fastapi" version = "0.139.2" requires_python = ">=3.10" summary = "FastAPI framework, high performance, easy to learn, fast to code, ready for production" -groups = ["default"] +groups = ["app"] dependencies = [ "annotated-doc>=0.0.2", "pydantic>=2.9.0", @@ -562,7 +562,7 @@ name = "feedparser" version = "6.0.14" requires_python = ">=3.10" summary = "Universal feed parser, handles RSS 0.9x, RSS 1.0, RSS 2.0, CDF, Atom 0.3, and Atom 1.0 feeds" -groups = ["default"] +groups = ["app"] dependencies = [ "feedparser-sgmllib<3,>=2", ] @@ -576,7 +576,7 @@ name = "feedparser-sgmllib" version = "2.1.0" requires_python = ">=3.10" summary = "sgmllib from Python 2.7. For feedparser use only." -groups = ["default"] +groups = ["app"] files = [ {file = "feedparser_sgmllib-2.1.0-py3-none-any.whl", hash = "sha256:2cab2d43b95a954f920f18aebce7a4dbbb3f539780b127e2aa114f579821e01d"}, {file = "feedparser_sgmllib-2.1.0.tar.gz", hash = "sha256:61facf2918c4389b5b00714f76c5e03431ffcd94cd1f51d657edd6cd7c396579"}, @@ -597,7 +597,7 @@ files = [ name = "filetype" version = "1.2.0" summary = "Infer file type and MIME type of any file/buffer. No external dependencies." -groups = ["default"] +groups = ["app"] files = [ {file = "filetype-1.2.0-py2.py3-none-any.whl", hash = "sha256:7ce71b6880181241cf7ac8697a2f1eb6a8bd9b429f7ad6d27b8db9ba5f1c2d25"}, {file = "filetype-1.2.0.tar.gz", hash = "sha256:66b56cd6474bf41d8c54660347d37afcc3f7d1970648de365c102ef77548aadb"}, @@ -608,7 +608,7 @@ name = "flupy" version = "1.2.4" requires_python = ">=3.10" summary = "Fluent data processing in Python - a chainable stream processing library for expressive data manipulation using method chaining" -groups = ["default"] +groups = ["app"] files = [ {file = "flupy-1.2.4-py3-none-any.whl", hash = "sha256:231192ded613ebde6d0435e9d1bf0e305cda3a5115be9a142ef368e3f5724f81"}, {file = "flupy-1.2.4.tar.gz", hash = "sha256:dad657113e4a271a322229dc2926c5154d4a926c6ba8154b604f7e9cdda5c3e6"}, @@ -660,7 +660,7 @@ name = "greenlet" version = "3.5.5" requires_python = ">=3.10" summary = "Lightweight in-process concurrent programming" -groups = ["default"] +groups = ["default", "app"] files = [ {file = "greenlet-3.5.5-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:49520f0c95a48b42cf55414b8e8479beb274ea70431afc33e3f79903c71f4380"}, {file = "greenlet-3.5.5-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:55272212cbc5f43d1d723725ab931f1939969b7e9523882ca58b55061769d053"}, @@ -680,7 +680,7 @@ name = "h11" version = "0.16.0" requires_python = ">=3.8" summary = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" -groups = ["default", "extension-preview", "extension-publisher"] +groups = ["default", "app", "extension-preview", "extension-publisher"] files = [ {file = "h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86"}, {file = "h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1"}, @@ -702,7 +702,7 @@ name = "htmldate" version = "1.10.0" requires_python = ">=3.10" summary = "Fast and robust extraction of original and updated publication dates from URLs and web pages." -groups = ["default"] +groups = ["app"] dependencies = [ "charset-normalizer>=3.4.0", "dateparser>=1.1.2", @@ -720,7 +720,7 @@ name = "httpcore" version = "1.0.9" requires_python = ">=3.8" summary = "A minimal low-level HTTP client." -groups = ["default", "extension-preview", "extension-publisher"] +groups = ["default", "app", "extension-preview", "extension-publisher"] dependencies = [ "certifi", "h11>=0.16", @@ -735,7 +735,7 @@ name = "httpcore2" version = "2.12.0" requires_python = ">=3.10" summary = "A minimal low-level HTTP client." -groups = ["default"] +groups = ["app"] marker = "sys_platform != \"emscripten\"" dependencies = [ "h11>=0.16", @@ -751,7 +751,7 @@ name = "httpx" version = "0.28.1" requires_python = ">=3.8" summary = "The next generation HTTP client." -groups = ["default", "extension-preview", "extension-publisher"] +groups = ["default", "app", "extension-preview", "extension-publisher"] dependencies = [ "anyio", "certifi", @@ -768,7 +768,7 @@ name = "httpx2" version = "2.12.0" requires_python = ">=3.10" summary = "The next generation HTTP client." -groups = ["default"] +groups = ["app"] dependencies = [ "anyio>=4.10; sys_platform != \"emscripten\"", "httpcore2==2.12.0; sys_platform != \"emscripten\"", @@ -787,7 +787,7 @@ name = "httpx2-jsfetch" version = "1.0" requires_python = ">=3.12" summary = "httpx2 transports for Emscripten/Pyodide, backed by the JavaScript fetch API." -groups = ["default"] +groups = ["app"] marker = "sys_platform == \"emscripten\" and python_version >= \"3.12\"" files = [ {file = "httpx2_jsfetch-1.0-py3-none-any.whl", hash = "sha256:cb916b707601e69a07721aabc8f3f6659be3a6893bc1ff5c6f9e02241df2da32"}, @@ -800,7 +800,7 @@ version = "0.28.1" extras = ["socks"] requires_python = ">=3.8" summary = "The next generation HTTP client." -groups = ["default"] +groups = ["app"] dependencies = [ "httpx==0.28.1", "socksio==1.*", @@ -826,7 +826,7 @@ name = "idna" version = "3.19" requires_python = ">=3.9" summary = "Internationalized Domain Names in Applications (IDNA)" -groups = ["default", "extension-preview", "extension-publisher"] +groups = ["default", "app", "extension-preview", "extension-publisher"] files = [ {file = "idna-3.19-py3-none-any.whl", hash = "sha256:815e7be7a7806d54abb586dc943addc79e8b2ee16915059658cbeff4b1b43bf4"}, {file = "idna-3.19.tar.gz", hash = "sha256:5e0811a4383b21dc5838069f801c4fb62113b7447663d2530d2bd6e77b49bf15"}, @@ -837,7 +837,7 @@ name = "imapclient" version = "3.1.0" requires_python = ">=3.8.0" summary = "Easy-to-use, Pythonic and complete IMAP client library" -groups = ["default"] +groups = ["app"] files = [ {file = "imapclient-3.1.0-py2.py3-none-any.whl", hash = "sha256:34c192ffe82936e940a998ac89c474296942cd7c4f6c208f0fccfcd972946038"}, {file = "imapclient-3.1.0.tar.gz", hash = "sha256:b0413a516ffcc4b4d69f2c611c0e95da57968edf5a5c78ac82d1bf882305facc"}, @@ -860,7 +860,7 @@ version = "0.2.0" requires_python = "<3.14,>=3.12" url = "https://github.com/InKCre/ext-reg/releases/download/runtime-core-py-v0.2.0/inkcre_extension_runtime_core_py-0.2.0-py3-none-any.whl" summary = "InKCre Core Python Extension Host Runtime" -groups = ["default"] +groups = ["app"] dependencies = [ "fastapi<0.142,>=0.139.2", "packaging<27,>=25", @@ -949,7 +949,7 @@ name = "joserfc" version = "1.7.4" requires_python = ">=3.10" summary = "The ultimate Python library for JOSE RFCs, including JWS, JWE, JWK, JWA, JWT" -groups = ["default"] +groups = ["app"] dependencies = [ "cryptography>=45.0.1", ] @@ -962,7 +962,7 @@ files = [ name = "js2py-3-13" version = "0.74.1" summary = "JavaScript to Python Translator & JavaScript interpreter written in 100% pure Python." -groups = ["default"] +groups = ["app"] dependencies = [ "pyjsparser>=2.5.1", "six>=1.10", @@ -978,7 +978,7 @@ name = "jsonschema" version = "4.26.0" requires_python = ">=3.10" summary = "An implementation of JSON Schema validation for Python" -groups = ["default"] +groups = ["default", "app"] dependencies = [ "attrs>=22.2.0", "jsonschema-specifications>=2023.03.6", @@ -995,7 +995,7 @@ name = "jsonschema-specifications" version = "2025.9.1" requires_python = ">=3.9" summary = "The JSON Schema meta-schemas and vocabularies, exposed as a Registry" -groups = ["default"] +groups = ["default", "app"] dependencies = [ "referencing>=0.31.0", ] @@ -1008,7 +1008,7 @@ files = [ name = "justext" version = "3.0.2" summary = "Heuristic based boilerplate removal tool" -groups = ["default"] +groups = ["app"] dependencies = [ "backports-functools-lru-cache; python_version < \"3.2\"", "lxml[html_clean]>=4.4.2", @@ -1023,7 +1023,7 @@ name = "libpass" version = "1.9.4" requires_python = ">=3.9" summary = "Fork of passlib, a comprehensive password hashing framework supporting over 30 schemes" -groups = ["default"] +groups = ["app"] files = [ {file = "libpass-1.9.4-py3-none-any.whl", hash = "sha256:a608621af2d7d64579957988fac287f230f88792c83a2d0f895afb05ee429489"}, {file = "libpass-1.9.4.tar.gz", hash = "sha256:4ff9a5ebf51aad1e8164453bc29c4a06926cf175a9501d87205612ba247f3cbd"}, @@ -1034,7 +1034,7 @@ name = "logtail-python" version = "0.4.0" requires_python = ">=3.10" summary = "Better Stack client library" -groups = ["default"] +groups = ["app"] dependencies = [ "msgpack>=0.5.6", "requests>=2.18.4", @@ -1049,7 +1049,7 @@ name = "lxml" version = "6.1.2" requires_python = ">=3.8" summary = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API." -groups = ["default"] +groups = ["default", "app"] files = [ {file = "lxml-6.1.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:7d506bdba580ecb1a6ad2e2b5c49445e66d3e1f95894885739094393a1aad237"}, {file = "lxml-6.1.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:12acd337d2821cb8b9247dfe4b7aa2f2769a3df5ae8511b7e550df42b8f4d3c3"}, @@ -1076,7 +1076,7 @@ files = [ name = "lxml-html-clean" version = "0.4.5" summary = "HTML cleaner from lxml project" -groups = ["default"] +groups = ["default", "app"] dependencies = [ "lxml>=6.1.1", ] @@ -1091,7 +1091,7 @@ version = "6.1.2" extras = ["html_clean"] requires_python = ">=3.8" summary = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API." -groups = ["default"] +groups = ["default", "app"] dependencies = [ "lxml-html-clean", "lxml==6.1.2", @@ -1123,7 +1123,7 @@ name = "m3u8" version = "6.0.0" requires_python = ">=3.7" summary = "Python m3u8 parser" -groups = ["default"] +groups = ["app"] dependencies = [ "backports-datetime-fromisoformat; python_version < \"3.11\"", ] @@ -1137,7 +1137,7 @@ name = "mako" version = "1.4.1" requires_python = ">=3.10" summary = "A super-fast templating language that borrows the best ideas from the existing templating languages." -groups = ["default"] +groups = ["app"] dependencies = [ "MarkupSafe>=2.0", ] @@ -1165,7 +1165,7 @@ name = "markupsafe" version = "3.0.3" requires_python = ">=3.9" summary = "Safely add untrusted strings to HTML/XML markup." -groups = ["default", "dev"] +groups = ["app", "dev"] files = [ {file = "markupsafe-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e"}, {file = "markupsafe-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce"}, @@ -1186,7 +1186,7 @@ name = "mcp" version = "2.1.1" requires_python = ">=3.10" summary = "Model Context Protocol SDK" -groups = ["default"] +groups = ["app"] dependencies = [ "anyio>=4.10; python_version >= \"3.14\"", "anyio>=4.9; python_version < \"3.14\"", @@ -1215,7 +1215,7 @@ name = "mcp-types" version = "2.1.1" requires_python = ">=3.10" summary = "Model Context Protocol wire types" -groups = ["default"] +groups = ["app"] dependencies = [ "pydantic>=2.12.0", "typing-extensions>=4.13.0", @@ -1241,7 +1241,7 @@ name = "msgpack" version = "1.2.1" requires_python = ">=3.10" summary = "MessagePack serializer" -groups = ["default"] +groups = ["app"] files = [ {file = "msgpack-1.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2ef59c659f289eddf8aa6623823f19fa2f40a4029266889eac7a2505dd210c35"}, {file = "msgpack-1.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d3567748a5107cb40cdf66a275430c2f87c07777698f4bfd25c35f44d533258c"}, @@ -1347,7 +1347,7 @@ name = "opentelemetry-api" version = "1.45.0" requires_python = ">=3.10" summary = "OpenTelemetry Python API" -groups = ["default"] +groups = ["default", "app"] dependencies = [ "typing-extensions>=4.5.0", ] @@ -1485,7 +1485,7 @@ name = "packaging" version = "25.0" requires_python = ">=3.8" summary = "Core utilities for Python packages" -groups = ["default", "dev", "extension-preview", "extension-publisher"] +groups = ["app", "dev", "extension-preview", "extension-publisher"] files = [ {file = "packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484"}, {file = "packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f"}, @@ -1495,7 +1495,7 @@ files = [ name = "parse" version = "1.22.1" summary = "parse() is the opposite of format()" -groups = ["default"] +groups = ["app"] files = [ {file = "parse-1.22.1-py2.py3-none-any.whl", hash = "sha256:20f0925a46f06602485ac90d751764d0697fd8455aaa97489ba8953a4b66de32"}, {file = "parse-1.22.1.tar.gz", hash = "sha256:d3a4740ec3da338e2b258b2d69741b731eadfddca59e24a14bc4ee5fce38c911"}, @@ -1520,7 +1520,7 @@ name = "pika" version = "1.4.4" requires_python = ">=3.7" summary = "Pika Python AMQP Client Library" -groups = ["default"] +groups = ["app"] files = [ {file = "pika-1.4.4-py3-none-any.whl", hash = "sha256:48de960c97a93b55db06b8be4c53eb977c9c8a2754c57cdae9097abcbd70ce04"}, {file = "pika-1.4.4.tar.gz", hash = "sha256:8cfc8b33a5cb16e733bd60cffca9732c0d1d761ecd80a89f34ed7df2cd38d6d6"}, @@ -1550,7 +1550,7 @@ name = "pip" version = "25.3" requires_python = ">=3.9" summary = "The PyPA recommended tool for installing Python packages." -groups = ["default"] +groups = ["app"] files = [ {file = "pip-25.3-py3-none-any.whl", hash = "sha256:9655943313a94722b7774661c21049070f6bbb0a1516bf02f7c8d5d9201514cd"}, {file = "pip-25.3.tar.gz", hash = "sha256:8d0538dbbd7babbd207f261ed969c65de439f6bc9e5dbd3b3b9a77f25d95f343"}, @@ -1709,7 +1709,7 @@ name = "pycparser" version = "3.0" requires_python = ">=3.10" summary = "C parser in Python" -groups = ["default"] +groups = ["app"] marker = "implementation_name != \"PyPy\" and platform_python_implementation != \"PyPy\"" files = [ {file = "pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992"}, @@ -1721,7 +1721,7 @@ name = "pydantic" version = "2.13.4" requires_python = ">=3.9" summary = "Data validation using Python type hints" -groups = ["default", "extension-preview", "extension-publisher"] +groups = ["default", "app", "extension-preview", "extension-publisher"] dependencies = [ "annotated-types>=0.6.0", "pydantic-core==2.46.4", @@ -1738,7 +1738,7 @@ name = "pydantic-core" version = "2.46.4" requires_python = ">=3.9" summary = "Core functionality for Pydantic validation and serialization" -groups = ["default", "extension-preview", "extension-publisher"] +groups = ["default", "app", "extension-preview", "extension-publisher"] dependencies = [ "typing-extensions>=4.14.1", ] @@ -1782,7 +1782,7 @@ name = "pygithub" version = "2.10.0" requires_python = ">=3.10" summary = "Use the full Github API v3" -groups = ["default"] +groups = ["app"] dependencies = [ "pyjwt[crypto]>=2.4.0", "pynacl>=1.4.0", @@ -1810,7 +1810,7 @@ files = [ name = "pyjsparser" version = "2.7.1" summary = "Fast javascript parser (based on esprima.js)" -groups = ["default"] +groups = ["app"] files = [ {file = "pyjsparser-2.7.1.tar.gz", hash = "sha256:be60da6b778cc5a5296a69d8e7d614f1f870faf94e1b1b6ac591f2ad5d729579"}, ] @@ -1820,7 +1820,7 @@ name = "pyjwt" version = "2.13.0" requires_python = ">=3.9" summary = "JSON Web Token implementation in Python" -groups = ["default"] +groups = ["default", "app"] dependencies = [ "typing-extensions>=4.0; python_version < \"3.11\"", ] @@ -1835,7 +1835,7 @@ version = "2.13.0" extras = ["crypto"] requires_python = ">=3.9" summary = "JSON Web Token implementation in Python" -groups = ["default"] +groups = ["app"] dependencies = [ "cryptography>=3.4.0", "pyjwt==2.13.0", @@ -1850,7 +1850,7 @@ name = "pynacl" version = "1.6.2" requires_python = ">=3.8" summary = "Python binding to the Networking and Cryptography (NaCl) library" -groups = ["default"] +groups = ["app"] dependencies = [ "cffi>=1.4.1; platform_python_implementation != \"PyPy\" and python_version < \"3.9\"", "cffi>=2.0.0; platform_python_implementation != \"PyPy\" and python_version >= \"3.9\"", @@ -1876,7 +1876,7 @@ name = "pyotp" version = "2.10.0" requires_python = ">=3.8" summary = "Python One Time Password Library" -groups = ["default"] +groups = ["app"] files = [ {file = "pyotp-2.10.0-py3-none-any.whl", hash = "sha256:1df2f6a1bcc3bb0716172a5215ddc2f8c7c7fd26a13df9927d52e1746934836c"}, {file = "pyotp-2.10.0.tar.gz", hash = "sha256:d01e9703443616b03c57c700b5cbffd56a1f929c1b0f8f03131bc78c1fca9d3f"}, @@ -1953,7 +1953,7 @@ name = "python-dateutil" version = "2.9.0.post0" requires_python = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" summary = "Extensions to the standard Python datetime module" -groups = ["default"] +groups = ["default", "app"] dependencies = [ "six>=1.5", ] @@ -1992,7 +1992,7 @@ name = "python-multipart" version = "0.0.32" requires_python = ">=3.10" summary = "A streaming multipart parser for Python" -groups = ["default"] +groups = ["app"] files = [ {file = "python_multipart-0.0.32-py3-none-any.whl", hash = "sha256:ff6d3f776f16878c894e52e107296ffc890e913c611b1a4ec6c44e2821fe2e23"}, {file = "python_multipart-0.0.32.tar.gz", hash = "sha256:be54b7f3fa167bb83e4fcd936b887b708f4e57fe75911c02aebf53efaf8d938e"}, @@ -2003,7 +2003,7 @@ name = "python-telegram-bot" version = "22.8" requires_python = ">=3.10" summary = "We have made you a wrapper you can't refuse" -groups = ["default"] +groups = ["app"] dependencies = [ "httpcore>=1.0.9; python_version >= \"3.14\"", "httpx<0.29,>=0.27", @@ -2017,7 +2017,7 @@ files = [ name = "pytz" version = "2026.3.post1" summary = "World timezone definitions, modern and historical" -groups = ["default"] +groups = ["app"] files = [ {file = "pytz-2026.3.post1-py2.py3-none-any.whl", hash = "sha256:dd95840dd199baea12d9cc096a1d452caa6596a1c1e4b5f3dbd1541855d5e815"}, {file = "pytz-2026.3.post1.tar.gz", hash = "sha256:2211d3fcf9a797d3405cac96ac7f61d80e6a644f72a3309607282fe8a2010c5d"}, @@ -2028,7 +2028,7 @@ name = "pywin32" version = "312" requires_python = ">=3.9" summary = "Python for Windows Extensions" -groups = ["default"] +groups = ["app"] marker = "sys_platform == \"win32\"" files = [ {file = "pywin32-312-cp312-cp312-win32.whl", hash = "sha256:dab4f65ac9c4e48400a2a0530c46c3c579cd5905ecd11b80692373915269208b"}, @@ -2061,7 +2061,7 @@ name = "radicale" version = "3.8.3" requires_python = ">=3.9.0" summary = "CalDAV and CardDAV Server" -groups = ["default"] +groups = ["app"] dependencies = [ "defusedxml", "libpass>=1.9.3", @@ -2095,7 +2095,7 @@ name = "referencing" version = "0.37.0" requires_python = ">=3.10" summary = "JSON Referencing + Python" -groups = ["default"] +groups = ["default", "app"] dependencies = [ "attrs>=22.2.0", "rpds-py>=0.7.0", @@ -2111,7 +2111,7 @@ name = "regex" version = "2026.7.19" requires_python = ">=3.10" summary = "Alternative regular expression module, to replace re." -groups = ["default"] +groups = ["app"] files = [ {file = "regex-2026.7.19-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:2cc3460cedf7579948486eab03bc9ad7089df4d7281c0f47f4afe03e8d13f02d"}, {file = "regex-2026.7.19-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0e9554c8785eac5cffe6300f69a91f58ba72bc88a5f8d661235ad7c6aa5b8ccd"}, @@ -2137,7 +2137,7 @@ name = "requests" version = "2.34.2" requires_python = ">=3.10" summary = "Python HTTP for Humans." -groups = ["default"] +groups = ["default", "app"] dependencies = [ "certifi>=2023.5.7", "charset-normalizer<4,>=2", @@ -2169,7 +2169,7 @@ name = "rpds-py" version = "2026.6.3" requires_python = ">=3.11" summary = "Python bindings to Rust's persistent data structures (rpds)" -groups = ["default"] +groups = ["default", "app"] files = [ {file = "rpds_py-2026.6.3-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a0811d33247c3d6128a3001d763f2aa056bb3425204335400ac54f89eec3a0d0"}, {file = "rpds_py-2026.6.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:538949e262e46caa31ac01bdb3c1e8f642622922cacbabbae6a8445d9dc33eaf"}, @@ -2221,7 +2221,7 @@ name = "semantic-version" version = "2.10.0" requires_python = ">=2.7" summary = "A library implementing the 'SemVer' scheme." -groups = ["default", "extension-preview", "extension-publisher"] +groups = ["app", "extension-preview", "extension-publisher"] files = [ {file = "semantic_version-2.10.0-py2.py3-none-any.whl", hash = "sha256:de78a3b8e0feda74cabc54aab2da702113e33ac9d9eb9d2389bcf1f58b7d9177"}, {file = "semantic_version-2.10.0.tar.gz", hash = "sha256:bdabb6d336998cbb378d4b9db3a4b56a1e3235701dc05ea2690d9a997ed5041c"}, @@ -2254,7 +2254,7 @@ name = "six" version = "1.17.0" requires_python = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" summary = "Python 2 and 3 compatibility utilities" -groups = ["default"] +groups = ["default", "app"] files = [ {file = "six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274"}, {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"}, @@ -2276,7 +2276,7 @@ name = "socksio" version = "1.0.0" requires_python = ">=3.6" summary = "Sans-I/O implementation of SOCKS4, SOCKS4A, and SOCKS5." -groups = ["default"] +groups = ["app"] files = [ {file = "socksio-1.0.0-py3-none-any.whl", hash = "sha256:95dc1f15f9b34e8d7b16f06d74b8ccf48f609af32ab33c608d08761c5dcbb1f3"}, {file = "socksio-1.0.0.tar.gz", hash = "sha256:f88beb3da5b5c38b9890469de67d0cb0f9d494b78b106ca1845f96c10b91c4ac"}, @@ -2287,7 +2287,7 @@ name = "soupsieve" version = "2.9.2" requires_python = ">=3.10" summary = "A modern CSS selector implementation for Beautiful Soup." -groups = ["default"] +groups = ["default", "app"] files = [ {file = "soupsieve-2.9.2-py3-none-any.whl", hash = "sha256:8089a26fd974ca7a1f30276d3d8492ab266ab15af581642dfe8aa162e0c1c823"}, {file = "soupsieve-2.9.2.tar.gz", hash = "sha256:4a55d8cf158a9c2e587fa4922f1bbb91d68ac829e2d6f25403a85747c71daf74"}, @@ -2298,7 +2298,7 @@ name = "sqlalchemy" version = "2.0.54" requires_python = ">=3.7" summary = "Database Abstraction Library" -groups = ["default"] +groups = ["default", "app"] dependencies = [ "greenlet>=1; platform_machine == \"win32\" or platform_machine == \"WIN32\" or platform_machine == \"AMD64\" or platform_machine == \"amd64\" or platform_machine == \"x86_64\" or platform_machine == \"ppc64le\" or platform_machine == \"aarch64\"", "importlib-metadata; python_version < \"3.8\"", @@ -2359,7 +2359,7 @@ name = "sse-starlette" version = "3.4.8" requires_python = ">=3.10" summary = "SSE plugin for Starlette" -groups = ["default"] +groups = ["app"] dependencies = [ "anyio>=4.7.0", "starlette>=0.49.1", @@ -2374,7 +2374,7 @@ name = "starlette" version = "1.6.0" requires_python = ">=3.10" summary = "The little ASGI library that shines." -groups = ["default"] +groups = ["app"] dependencies = [ "anyio<5,>=3.6.2", "typing-extensions>=4.10.0; python_version < \"3.13\"", @@ -2415,7 +2415,7 @@ name = "tld" version = "0.13.2" requires_python = ">=3.7" summary = "Extract the top-level domain (TLD) from the URL given." -groups = ["default"] +groups = ["app"] files = [ {file = "tld-0.13.2-py2.py3-none-any.whl", hash = "sha256:9b8fdbdb880e7ba65b216a4937f2c94c49a7226723783d5838fc958ac76f4e0c"}, {file = "tld-0.13.2.tar.gz", hash = "sha256:d983fa92b9d717400742fca844e29d5e18271079c7bcfabf66d01b39b4a14345"}, @@ -2458,7 +2458,7 @@ name = "trafilatura" version = "2.2.0" requires_python = ">=3.10" summary = "Python & Command-line tool to gather text and metadata on the Web: Crawling, scraping, extraction, output as CSV, JSON, HTML, MD, TXT, XML." -groups = ["default"] +groups = ["app"] dependencies = [ "certifi", "charset-normalizer>=3.4.9", @@ -2478,7 +2478,7 @@ name = "truststore" version = "0.10.4" requires_python = ">= 3.10" summary = "Verify certificates using native system trust stores" -groups = ["default"] +groups = ["app"] marker = "sys_platform != \"emscripten\"" files = [ {file = "truststore-0.10.4-py3-none-any.whl", hash = "sha256:adaeaecf1cbb5f4de3b1959b42d41f6fab57b2b1666adb59e89cb0b53361d981"}, @@ -2490,7 +2490,7 @@ name = "twikit" version = "2.3.3" requires_python = ">=3.8" summary = "Twitter API wrapper for python with **no API key required**." -groups = ["default"] +groups = ["app"] dependencies = [ "Js2Py-3-13", "beautifulsoup4", @@ -2528,7 +2528,7 @@ name = "typing-extensions" version = "4.16.0" requires_python = ">=3.9" summary = "Backported and Experimental Type Hints for Python 3.9+" -groups = ["default", "extension-preview", "extension-publisher"] +groups = ["default", "app", "extension-preview", "extension-publisher"] files = [ {file = "typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8"}, {file = "typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5"}, @@ -2539,7 +2539,7 @@ name = "typing-inspection" version = "0.4.4" requires_python = ">=3.10" summary = "Runtime typing introspection tools" -groups = ["default", "extension-preview", "extension-publisher"] +groups = ["default", "app", "extension-preview", "extension-publisher"] dependencies = [ "typing-extensions>=4.15.0", ] @@ -2553,7 +2553,7 @@ name = "tzdata" version = "2026.3" requires_python = ">=2" summary = "Provider of IANA time zone data" -groups = ["default"] +groups = ["default", "app"] marker = "sys_platform == \"win32\" or platform_system == \"Windows\"" files = [ {file = "tzdata-2026.3-py2.py3-none-any.whl", hash = "sha256:dc096730c87af6cab1b171c9d532be840741ff5d459015e7f6947bd7d7e54931"}, @@ -2565,7 +2565,7 @@ name = "tzlocal" version = "5.4.4" requires_python = ">=3.10" summary = "tzinfo object for the local timezone" -groups = ["default"] +groups = ["app"] dependencies = [ "tzdata; platform_system == \"Windows\"", ] @@ -2579,7 +2579,7 @@ name = "urllib3" version = "2.7.0" requires_python = ">=3.10" summary = "HTTP library with thread-safe connection pooling, file post, and more." -groups = ["default"] +groups = ["default", "app"] files = [ {file = "urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897"}, {file = "urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c"}, @@ -2618,7 +2618,7 @@ name = "uvicorn" version = "0.35.0" requires_python = ">=3.9" summary = "The lightning-fast ASGI server." -groups = ["default"] +groups = ["app"] dependencies = [ "click>=7.0", "h11>=0.8", @@ -2652,7 +2652,7 @@ files = [ name = "vobject" version = "0.9.9" summary = "A full-featured Python package for parsing and creating iCalendar and vCard files" -groups = ["default"] +groups = ["app"] dependencies = [ "python-dateutil>=2.5.0; python_version < \"3.10\"", "python-dateutil>=2.7.0; python_version >= \"3.10\"", @@ -2669,7 +2669,7 @@ name = "webvtt-py" version = "0.5.1" requires_python = ">=3.7" summary = "WebVTT reader, writer and segmenter" -groups = ["default"] +groups = ["app"] files = [ {file = "webvtt-py-0.5.1.tar.gz", hash = "sha256:2040dd325277ddadc1e0c6cc66cbc4a1d9b6b49b24c57a0c3364374c3e8a3dc1"}, {file = "webvtt_py-0.5.1-py3-none-any.whl", hash = "sha256:9d517d286cfe7fc7825e9d4e2079647ce32f5678eb58e39ef544ffbb932610b7"}, diff --git a/pyproject.toml b/pyproject.toml index 44f6581b..dcfd5673 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,19 +1,13 @@ [project] name = "inkcre-core" version = "0.10.0" -description = "Backend of InKCre." +description = "InKCre Python SDK for information-base operations and Peer capabilities." authors = [{ name = "Lan_zhijiang", email = "lanzhijiang@foxmail.com" }] readme = "README.md" requires-python = ">=3.12,<3.13" dependencies = [ - "inkcre-extension-runtime-core-py @ https://github.com/InKCre/ext-reg/releases/download/runtime-core-py-v0.2.0/inkcre_extension_runtime_core_py-0.2.0-py3-none-any.whl", - "authlib>=1.7.2,<2.0.0", "pydantic (>=2.11.7,<3.0.0)", "pydantic-settings (>=2.14.2,<3.0.0)", - "fastapi (>=0.139.2,<0.140.0)", - "packaging (>=25.0,<26.0)", - "pip (>=25.2,<26.0)", - "semantic-version (>=2.10.0,<3.0.0)", "sqlalchemy[asyncio]<3.0.0,>=2.0.41", "aiohttp (>=3.14.1,<4.0.0)", "tencentcloud-sdk-python-lke (>=3.0.1427,<4.0.0)", @@ -21,20 +15,12 @@ dependencies = [ "openai (>=1.97.1,<2.0.0)", "numpy (>=2.3.2,<3.0.0)", "pgvector (>=0.4.1,<0.5.0)", - "uvicorn (>=0.35.0,<0.36.0)", "requests (>=2.33.0,<3.0.0)", "httpx>=0.28.1,<0.29.0", "sqlmodel (>=0.0.24,<0.0.25)", "datadot (>=1.0.0,<2.0.0)", - "apscheduler>=3.11.0", "html2text>=2025.4.15", - "logtail-python>=0.3.0", - "python-telegram-bot==22.8", - "alembic>=1.16.4", - "alembic-utils>=0.8.8", - "alembic-postgresql-enum>=1.8.0", "PyJWT>=2.10.1", - # Fixed built-in extension profile "beautifulsoup4>=4.12.0,<5.0.0", "chardet>=5.0.0,<6.0.0", "lxml>=5.0.0,<7.0.0", @@ -43,19 +29,36 @@ dependencies = [ "av>=18.0.0,<19.0.0", "pypdf>=6.14.2,<7.0.0", "puremagic>=2.2.0,<3.0.0", + "jsonschema<5,>=4.25", + "croniter<7,>=6", + "opentelemetry-sdk==1.45.0", + "opentelemetry-exporter-otlp-proto-http==1.45.0", + "anyio<5,>=4", +] + +[project.optional-dependencies] +app = [ + "inkcre-extension-runtime-core-py @ https://github.com/InKCre/ext-reg/releases/download/runtime-core-py-v0.2.0/inkcre_extension_runtime_core_py-0.2.0-py3-none-any.whl", + "authlib>=1.7.2,<2.0.0", + "fastapi (>=0.139.2,<0.140.0)", + "packaging (>=25.0,<26.0)", + "pip (>=25.2,<26.0)", + "semantic-version (>=2.10.0,<3.0.0)", + "uvicorn (>=0.35.0,<0.36.0)", + "apscheduler>=3.11.0", + "logtail-python>=0.3.0", + "python-telegram-bot==22.8", + "alembic>=1.16.4", + "alembic-utils>=0.8.8", + "alembic-postgresql-enum>=1.8.0", "feedparser>=6.0.12,<7.0.0", "trafilatura>=2.1.0,<3.0.0", "PyGithub>=2.8.1,<3.0.0", "twikit>=2.3.3,<3.0.0", - "jsonschema<5,>=4.25", - "croniter<7,>=6", "imapclient<4,>=3.1", "mcp>=2.1.1,<2.2.0", - "opentelemetry-sdk==1.45.0", - "opentelemetry-exporter-otlp-proto-http==1.45.0", "Radicale==3.8.3", "vobject<0.10,>=0.9.9", - "anyio<5,>=4", ] [dependency-groups] @@ -81,11 +84,8 @@ dev = [ "towncrier==25.8.0", ] -[tool.poetry] -package-mode = false - [tool.pdm] -distribution = false +distribution = true [tool.pdm.scripts] dev = "python run.py" @@ -107,11 +107,12 @@ test = "python -m pytest -q" "check:releases" = "python scripts/release.py check {args}" "build:extension-preview" = "python scripts/build_extension_preview.py {args}" "release:prepare" = "python scripts/release.py prepare {args}" +"check:core-package" = { composite = ["python -m build --wheel --no-isolation", "python scripts/check_core_package.py"] } "check:foundation" = { composite = ["doctor", "check:lock", "check:migrations", "check:automation"] } "lint:database-imports" = "ruff check --config ruff.database.toml ." "lint:database-ownership" = "python scripts/check_database_boundaries.py" "lint:database-boundaries" = { composite = ["lint:database-imports", "lint:database-ownership"] } -"check" = { composite = ["check:foundation", "format:check", "lint", "lint:database-boundaries", "typecheck", "test"] } +"check" = { composite = ["check:foundation", "format:check", "lint", "lint:database-boundaries", "typecheck", "check:core-package", "test"] } "db:revision" = "alembic revision -m {args}" "db:generate" = "alembic revision --autogenerate -m {args}" "db:record" = "python scripts/check_migration_history.py --record-new" @@ -135,6 +136,7 @@ test = "python -m pytest -q" [tool.pyrefly] project-includes = [ + "inkcre_core", "app", "extensions", "libs", @@ -159,5 +161,11 @@ markers = [ ] [build-system] -requires = [] -build-backend = "none" +requires = ["setuptools>=80,<81"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.packages.find] +include = ["inkcre_core*"] + +[tool.setuptools.package-data] +inkcre_core = ["py.typed"] diff --git a/ruff.database.toml b/ruff.database.toml index 498d7cf3..40003697 100644 --- a/ruff.database.toml +++ b/ruff.database.toml @@ -1,6 +1,6 @@ # Runtime-wide database rules. Readiness/CLI contract adapters own separate connections. extend = "ruff.toml" -include = ["app/**/*.py", "extensions/**/*.py", "libs/**/*.py", "run.py"] +include = ["app/**/*.py", "inkcre_core/**/*.py", "extensions/**/*.py", "libs/**/*.py", "run.py"] extend-exclude = ["app/database_contract"] [lint] @@ -9,7 +9,7 @@ select = ["TID251"] [lint.flake8-tidy-imports.banned-api] "sqlmodel.Session".msg = "运行时持久化使用绑定 UoW 的 AsyncSession。" "sqlalchemy.orm.Session".msg = "运行时持久化使用绑定 UoW 的 AsyncSession。" -"sqlalchemy.orm.sessionmaker".msg = "运行时只允许 app.engine 的异步工厂。" +"sqlalchemy.orm.sessionmaker".msg = "运行时只允许 inkcre_core.context 的异步工厂。" "sqlalchemy.orm.scoped_session".msg = "Session 生命周期必须显式，不使用 scoped session。" "sqlalchemy.ext.asyncio.async_scoped_session".msg = "每个 task 独立 UoW，不隐藏 session。" "sqlmodel.create_engine".msg = "运行时不创建同步 engine。" diff --git a/ruff.toml b/ruff.toml index d1b4b67b..54f9eb8a 100644 --- a/ruff.toml +++ b/ruff.toml @@ -2,37 +2,37 @@ line-length = 92 indent-width = 2 # Task probes are disposable evidence; maintained checks belong in scripts/ or tests/. extend-exclude = ["migrations/versions", "*.md", "cli", ".agents", "tasks"] - + [format] indent-style = "space" line-ending = "lf" - -[lint] -select = [ - "E501", # line length limit - "F821", # undefined-name - "B006", # mutable-argument-default - "E722", # bare-except - "PLE0101", # return-in-init (Pylint) - "RUF006", # asyncio-dangling-task - - "S105", # hardcoded-password-string (Bandit) - "S301", # suspicious-pickle-usage (Bandit) - "S307", # suspicious-eval-usage (Bandit) - "S602", # subprocess-popen-with-shell-equals-true (Bandit) - "S603", # subprocess-without-shell-equals-true (Bandit) - - "F401", # unused-import - - "PLR0913", # too-many-arguments (Pylint) - - "UP007", # non-pep604-annotation (pyupgrade) - "UP032", # f-string (pyupgrade) - - "SIM115", # open-file-with-context-handler (flake8-simplify) - "PLW1510", # subprocess-run-without-check (Pylint) - "RUF100", # unused-noqa -] - -[lint.pycodestyle] + +[lint] +select = [ + "E501", # line length limit + "F821", # undefined-name + "B006", # mutable-argument-default + "E722", # bare-except + "PLE0101", # return-in-init (Pylint) + "RUF006", # asyncio-dangling-task + + "S105", # hardcoded-password-string (Bandit) + "S301", # suspicious-pickle-usage (Bandit) + "S307", # suspicious-eval-usage (Bandit) + "S602", # subprocess-popen-with-shell-equals-true (Bandit) + "S603", # subprocess-without-shell-equals-true (Bandit) + + "F401", # unused-import + + "PLR0913", # too-many-arguments (Pylint) + + "UP007", # non-pep604-annotation (pyupgrade) + "UP032", # f-string (pyupgrade) + + "SIM115", # open-file-with-context-handler (flake8-simplify) + "PLW1510", # subprocess-run-without-check (Pylint) + "RUF100", # unused-noqa +] + +[lint.pycodestyle] max-line-length = 92 diff --git a/run.py b/run.py index 69daf633..63ca9b97 100644 --- a/run.py +++ b/run.py @@ -12,9 +12,11 @@ from fastapi.responses import JSONResponse # Setup logging -from libs.obsrv.main import close_obsrv, setup_obsrv, start_obsrv +from inkcre_core.libs.obsrv.main import close_obsrv, setup_obsrv, start_obsrv -logger = setup_obsrv() +from app.settings import settings + +logger = setup_obsrv(settings.obsrv) # Setup logging end # Load flags @@ -26,8 +28,7 @@ # Load flags end from fastapi.middleware.cors import CORSMiddleware -from app.settings import settings -from app.engine import ASYNC_DB_ENGINE +from app.core import CORE, CoreMiddleware from app.routes.block import ROUTER as block_router from app.routes.relation import ROUTER as relation_router from app.routes.extension import PEER_INBOUND as extension_peer_inbound @@ -52,19 +53,19 @@ from app.routes.semantic_retrieval import ROUTER as semantic_retrieval_router from app.routes.sink import ROUTER as sink_router from app.routes.telemetry import ROUTER as telemetry_router -from app.business.source import SourceManager -from app.business.cron import CronManager -from app.business.job import JobManager +from inkcre_core.business.source import SourceManager +from inkcre_core.business.cron import CronManager +from inkcre_core.business.job import JobManager from app.business.extension import EXTENSION_HOST -from app.business.peer import PeerManager -from app.business.ai import AIManager +from inkcre_core.business.peer import PeerManager +from inkcre_core.business.ai import AIManager from app.business.sink import SinkManager # Import core-owned Job contracts before their catalog is synchronized. -from app.business.organization_job import MediaInterpretationJobHandler # noqa: F401 +from inkcre_core.business.organization_job import MediaInterpretationJobHandler # noqa: F401 # Import independent Organization Job contracts before catalog synchronization. -from app.business.organization.jobs import ( # noqa: F401 +from inkcre_core.business.organization.jobs import ( # noqa: F401 DuplicateAssertionJobHandler, EvidenceStanceJobHandler, ExistingReferentAnchoringJobHandler, @@ -75,7 +76,7 @@ ) from app.middleware import LoggingMiddleware, require_peer_jwt from app.observability import TelemetryMiddleware, initialize_telemetry -from app.schemas.peer import PEER_EXECUTION_HEADER +from inkcre_core.schemas.peer import PEER_EXECUTION_HEADER from app.health import check_database_readiness from app.runtime import ( RUNTIME_STATUS, @@ -137,10 +138,10 @@ def bootstrap_step(step: str): async def bootstrap_runtime(app: fastapi.FastAPI) -> None: """Initialize database-backed runtime services after migrations are ready.""" - from app.business.info_base.resolver import register_core_resolvers - from app.business.agent import register_core_agent_tools - from app.business.organization import register_core_organization_behaviors - from app.business.info_base.storage import StorageManager + from inkcre_core.business.info_base.resolver import register_core_resolvers + from inkcre_core.business.agent import register_core_agent_tools + from inkcre_core.business.organization import register_core_organization_behaviors + from inkcre_core.business.info_base.storage import StorageManager # Register this Peer first so extension enablement can resolve its identity. with bootstrap_step("peer_registration"): @@ -282,7 +283,7 @@ async def lifespan(app: fastapi.FastAPI): try: await close_obsrv() finally: - await ASYNC_DB_ENGINE.dispose() + await CORE.aclose() api_app = fastapi.FastAPI(title="InKCre", lifespan=lifespan) @@ -330,6 +331,10 @@ async def peer_execution_gate(request: fastapi.Request, call_next): ) +# Bind outer request middleware and lifespan to the same owned instance. +api_app.add_middleware(CoreMiddleware, core=CORE) + + # Set up routes @api_app.get("/livez") def liveness() -> dict[str, str]: diff --git a/scripts/automation/production_preflight.py b/scripts/automation/production_preflight.py index 57306c0a..05bdd12d 100644 --- a/scripts/automation/production_preflight.py +++ b/scripts/automation/production_preflight.py @@ -8,7 +8,7 @@ from urllib.parse import unquote, urlsplit from app.database_contract.connection import database_connection -from app.database_contract.constants import ( +from inkcre_core.database_contract.constants import ( ANONYMOUS_ROLE, AUTHENTICATED_ROLE, AUTHENTICATOR_ROLE, diff --git a/scripts/benchmark_database.py b/scripts/benchmark_database.py index da015047..c88e8fe5 100644 --- a/scripts/benchmark_database.py +++ b/scripts/benchmark_database.py @@ -23,11 +23,11 @@ ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) -from app.business.info_base.commands import submit_graph -from app.business.info_base.services import get_entity_records +from inkcre_core.business.info_base.commands import submit_graph +from inkcre_core.business.info_base.services import get_entity_records from app.engine import ASYNC_DB_ENGINE, AsyncSessionFactory -from app.schemas.info_base.block import BlockModel -from app.schemas.info_base.main import GraphBlockForm, GraphForm, GraphRelationForm +from inkcre_core.schemas.info_base.block import BlockModel +from inkcre_core.schemas.info_base.main import GraphBlockForm, GraphForm, GraphRelationForm def positive(value: str) -> int: diff --git a/scripts/check_core_package.py b/scripts/check_core_package.py new file mode 100644 index 00000000..ef3c1103 --- /dev/null +++ b/scripts/check_core_package.py @@ -0,0 +1,77 @@ +"""Consume the built SDK wheel outside the checkout, without application imports.""" + +import ast +from pathlib import Path +import os +import subprocess +import sys +import tempfile +import zipfile + + +ROOT = Path(__file__).resolve().parents[1] +PROBE = """ +import asyncio +import importlib.abc +import sys + +class NoApplication(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + if fullname.split('.')[0] in {'app', 'fastapi', 'starlette', 'apscheduler'}: + raise AssertionError(f"SDK imported application dependency: {fullname}") + +sys.meta_path.insert(0, NoApplication()) +from inkcre_core import Core +from inkcre_core.business.info_base import BlockService, submit_graph +from inkcre_core.schemas.info_base.main import GraphForm +from inkcre_core.business.peer import PeerManager +from inkcre_core.business.job import JobManager + +async def main(): + core = Core('postgresql+psycopg://unused:x@127.0.0.1:1/unused') + with core.bind(): + assert PeerManager.get_current_peer_ref() == core.options.peer_id + assert not core.accepting_jobs + assert not core.pending_jobs + await core.aclose() + +asyncio.run(main()) +print('Independent wheel import and lifecycle passed; no database contacted') +""" + + +def main() -> None: + wheels = list((ROOT / "dist").glob("inkcre_core-*.whl")) + if len(wheels) != 1: + raise SystemExit("Build exactly one current SDK wheel in dist/ first") + with tempfile.TemporaryDirectory(prefix="inkcre-sdk-consumer-") as directory: + with zipfile.ZipFile(wheels[0]) as wheel: + assert all( + name.startswith(("inkcre_core/", "inkcre_core-")) for name in wheel.namelist() + ), "SDK wheel contains App source" + for name in wheel.namelist(): + if not name.endswith(".py"): + continue + for node in ast.walk(ast.parse(wheel.read(name))): + modules = [] + if isinstance(node, ast.Import): + modules = [alias.name for alias in node.names] + elif isinstance(node, ast.ImportFrom) and not node.level: + modules = [node.module or ""] + assert not any( + module.split(".")[0] in {"app", "fastapi", "starlette", "apscheduler"} + for module in modules + ), f"Application dependency in SDK: {name}" + wheel.extractall(directory) + environment = { + key: value + for key, value in os.environ.items() + if key not in {"PYTHONPATH", "DATABASE_URL", "JWT_SECRET", "INKCRE_ENV_FILE"} + } + subprocess.run( # noqa: S603 - fixed probe in the current Python environment. + [sys.executable, "-c", PROBE], cwd=directory, env=environment, check=True + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/check_database_boundaries.py b/scripts/check_database_boundaries.py index 480e898b..21cf6a0a 100644 --- a/scripts/check_database_boundaries.py +++ b/scripts/check_database_boundaries.py @@ -19,11 +19,11 @@ def inspect_source(path: str, source: str) -> list[str]: tree = ast.parse(source) imports: dict[str, str] = {} errors: list[str] = [] - persistence = path.startswith("app/persistence/") + persistence = path.startswith("inkcre_core/persistence/") uow = persistence and path.endswith("/uow.py") - engine = path == "app/engine.py" + engine = path == "inkcre_core/context.py" repository = persistence and not uow - business = path.startswith(("app/business/", "extensions/")) + business = path.startswith(("app/business/", "inkcre_core/business/", "extensions/")) route = path.startswith("app/routes/") def report(node: ast.stmt | ast.expr, message: str) -> None: @@ -59,9 +59,9 @@ def qualified(node: ast.AST) -> str: for name in names: if name.startswith("app.engine") and not (uow or path == "run.py"): report(node, "数据库工厂只能由 UoW 使用；lifespan 仅拥有 engine 释放。") - if route and name.startswith("app.persistence"): + if route and name.startswith(("inkcre_core.persistence", "app.persistence")): report(node, "HTTP route 必须调用 business，不直接依赖 persistence。") - if persistence and name.startswith("app.business"): + if persistence and name.startswith(("inkcre_core.business", "app.business")): report(node, "persistence 不得反向依赖 business。") if (business or route) and name.startswith(DB_MODULES): if name.rsplit(".", 1)[-1] in SESSION_CONSTRUCTORS | SQL_OPERATIONS: @@ -88,7 +88,7 @@ def qualified(node: ast.AST) -> str: leaf = name.rsplit(".", 1)[-1] if name.startswith(DB_MODULES): if leaf in SESSION_CONSTRUCTORS | ENGINE_CONSTRUCTORS and not engine: - report(node, "Engine 和 session factory 只在 app.engine 创建。") + report(node, "Engine 和 session factory 只在 inkcre_core.context 创建。") if (business or route) and leaf in SQL_OPERATIONS: report(node, "SQL 查询与写入必须位于 persistence。") if repository and isinstance(node.func, ast.Attribute): @@ -108,7 +108,7 @@ def self_check() -> None: assert inspect_source("app/routes/example.py", "from ..persistence.info_base import uow") assert inspect_source("app/business/example.py", "import sqlalchemy as sa\nsa.select(1)") assert inspect_source( - "app/persistence/x/repository.py", + "inkcre_core/persistence/x/repository.py", """ from sqlalchemy.ext.asyncio import AsyncSession as Session async def save(session: Session): @@ -117,7 +117,7 @@ async def save(session: Session): """, ) assert not inspect_source( - "app/persistence/x/repository.py", + "inkcre_core/persistence/x/repository.py", """ from sqlalchemy.ext.asyncio import AsyncSession async def save(session: AsyncSession): @@ -125,11 +125,11 @@ async def save(session: AsyncSession): """, ) assert not inspect_source( - "app/persistence/x/uow.py", + "inkcre_core/persistence/x/uow.py", """ -from app.engine import AsyncSessionFactory +from inkcre_core.context import current_core async def operation(): - async with AsyncSessionFactory.begin() as session: + async with current_core().sessions.begin() as session: yield session """, ) diff --git a/scripts/configure_peer_runtime.py b/scripts/configure_peer_runtime.py index c7bbd5f4..fd29536c 100644 --- a/scripts/configure_peer_runtime.py +++ b/scripts/configure_peer_runtime.py @@ -17,7 +17,7 @@ sys.path.insert(0, str(PROJECT_ROOT)) from app.database_contract.connection import database_connection -from app.schemas.peer import CorePeerConfig, PEER_HTTP_PROTOCOL +from inkcre_core.schemas.peer import CorePeerConfig, PEER_HTTP_PROTOCOL CAPABILITY_PATHS = { diff --git a/scripts/container.py b/scripts/container.py index 1a22c5a8..02704e08 100644 --- a/scripts/container.py +++ b/scripts/container.py @@ -28,7 +28,16 @@ def _web() -> int: from app.business.extension.main import EXTENSION_HOST - asyncio.run(EXTENSION_HOST.provision_enabled_distributions()) + from app.core import create_core + + async def provision() -> None: + with create_core().bind() as core: + try: + await EXTENSION_HOST.provision_enabled_distributions() + finally: + await core.aclose() + + asyncio.run(provision()) os.execvp( sys.executable, diff --git a/scripts/database_manifest.py b/scripts/database_manifest.py index bd3edc04..c3981a83 100644 --- a/scripts/database_manifest.py +++ b/scripts/database_manifest.py @@ -13,7 +13,7 @@ from sqlalchemy.pool import NullPool from migrations.settings import MigrationSettings -from app.database_contract.constants import PROTOCOL_SCHEMA +from inkcre_core.database_contract.constants import PROTOCOL_SCHEMA from scripts.sanitize_preview_base import LINEAGE_TABLE diff --git a/scripts/deploy_render_neon.py b/scripts/deploy_render_neon.py index 30a957fd..fb0effae 100644 --- a/scripts/deploy_render_neon.py +++ b/scripts/deploy_render_neon.py @@ -21,7 +21,7 @@ PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, PROJECT_ROOT) -from app.database_contract.constants import JWT_MINIMUM_SECRET_BYTES +from inkcre_core.database_contract.constants import JWT_MINIMUM_SECRET_BYTES from app.database_contract.lifecycle import initialize from app.database_contract.readiness import check_database_contract from app.database_contract.roles import RoleSecrets diff --git a/scripts/dev_database.py b/scripts/dev_database.py index 61535fd0..a8ed2899 100644 --- a/scripts/dev_database.py +++ b/scripts/dev_database.py @@ -20,7 +20,7 @@ PROJECT_ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(PROJECT_ROOT)) -from app.database_contract.constants import ( +from inkcre_core.database_contract.constants import ( CONTRACT_REVISION, JWT_ALGORITHM, JWT_AUDIENCE, diff --git a/scripts/export_database_roles.py b/scripts/export_database_roles.py index 05473ca0..91011f55 100644 --- a/scripts/export_database_roles.py +++ b/scripts/export_database_roles.py @@ -10,7 +10,7 @@ sys.path.insert(0, str(PROJECT_ROOT)) from app.database_contract.connection import database_connection -from app.database_contract.constants import ( +from inkcre_core.database_contract.constants import ( ANONYMOUS_ROLE, AUTHENTICATED_ROLE, AUTHENTICATOR_ROLE, diff --git a/scripts/observability.py b/scripts/observability.py index a6a6b8cb..d20b8d4a 100644 --- a/scripts/observability.py +++ b/scripts/observability.py @@ -9,7 +9,7 @@ from psycopg.types.json import Jsonb from app.database_contract.connection import database_connection -from app.schemas.observability import CONFIG_KEY, CONFIG_SCHEMA, ObservabilityConfig +from inkcre_core.schemas.observability import CONFIG_KEY, CONFIG_SCHEMA, ObservabilityConfig def main() -> None: diff --git a/scripts/sanitize_preview_base.py b/scripts/sanitize_preview_base.py index b4157926..213d96f3 100644 --- a/scripts/sanitize_preview_base.py +++ b/scripts/sanitize_preview_base.py @@ -15,7 +15,7 @@ from migrations.metadata import get_target_metadata from migrations.settings import MigrationSettings -from app.database_contract import INTERNAL_SCHEMA, PROTOCOL_SCHEMA +from inkcre_core.database_contract import INTERNAL_SCHEMA, PROTOCOL_SCHEMA SANITIZE_GUARD = "ALLOW_PREVIEW_BASE_SANITIZE" diff --git a/scripts/tests/caldav/conftest.py b/scripts/tests/caldav/conftest.py index b0730929..ba953049 100644 --- a/scripts/tests/caldav/conftest.py +++ b/scripts/tests/caldav/conftest.py @@ -1,3 +1,5 @@ """Reuse the isolated environment for explicitly selected CalDAV script checks.""" from tests import conftest # noqa: F401 + +from tests.conftest import application_core_scope as application_core_scope diff --git a/scripts/tests/caldav/test_dav_graph.py b/scripts/tests/caldav/test_dav_graph.py index 82f99a79..67936606 100644 --- a/scripts/tests/caldav/test_dav_graph.py +++ b/scripts/tests/caldav/test_dav_graph.py @@ -11,8 +11,8 @@ from sqlalchemy.orm import Session from app.engine import ASYNC_DB_ENGINE -from app.persistence.info_base.uow import graph_uow -from app.business.info_base.resolver import ResolverManager +from inkcre_core.persistence.info_base.uow import graph_uow +from inkcre_core.business.info_base.resolver import ResolverManager from extensions.caldav import Extension from extensions.caldav.schema import CALENDAR, GraphContent from tests.extensions.runtime_support import publish_extension @@ -201,7 +201,7 @@ def record_write(session, flush_context): def test_tasks_contacts_and_credential_revocation(dav): - from app.schemas.extension import ExtensionModel + from inkcre_core.schemas.extension import ExtensionModel from tests.database import TestSession task_path = f"/caldav/inkcre/tasks/{uuid4().hex}.ics" @@ -313,8 +313,8 @@ def test_collection_delete_serializes_direct_peer_membership_writes( import sqlmodel from sqlalchemy.exc import OperationalError - from app.schemas.info_base.block import BlockModel - from app.schemas.info_base.relation import RelationModel + from inkcre_core.schemas.info_base.block import BlockModel + from inkcre_core.schemas.info_base.relation import RelationModel from extensions.caldav.graph import DAVGraph from extensions.caldav.schema import COLLECTION, CollectionContent from tests.database import TestSession diff --git a/scripts/tests/caldav/test_semantic_graph.py b/scripts/tests/caldav/test_semantic_graph.py index a6ac2531..c15b89cb 100644 --- a/scripts/tests/caldav/test_semantic_graph.py +++ b/scripts/tests/caldav/test_semantic_graph.py @@ -12,15 +12,15 @@ import vobject # pyrefly: ignore[untyped-import] from sqlalchemy.exc import OperationalError -from app.business.info_base.resolver import ResolverManager -from app.business.info_base.storage import StorageManager -from app.business.info_base.storage.postgresql import ( +from inkcre_core.business.info_base.resolver import ResolverManager +from inkcre_core.business.info_base.storage import StorageManager +from inkcre_core.business.info_base.storage.postgresql import ( PostgreSQLBinaryStorage, PostgreSQLBlobPointer, ) -from app.persistence.info_base.uow import graph_uow -from app.schemas.info_base.block import BlockForm, BlockModel -from app.schemas.info_base.storage import StorageBlobModel +from inkcre_core.persistence.info_base.uow import graph_uow +from inkcre_core.schemas.info_base.block import BlockForm, BlockModel +from inkcre_core.schemas.info_base.storage import StorageBlobModel from extensions.caldav.graph import DAVGraph, migrate_legacy_graph from extensions.caldav.resolver import solve_resource from extensions.caldav.schema import ( diff --git a/scripts/tests/twitter/conftest.py b/scripts/tests/twitter/conftest.py index 3015882b..c67f1943 100644 --- a/scripts/tests/twitter/conftest.py +++ b/scripts/tests/twitter/conftest.py @@ -1,3 +1,5 @@ """Reuse the existing hermetic environment for explicitly selected Twitter checks.""" from tests.conftest import async_runner as async_runner + +from tests.conftest import application_core_scope as application_core_scope diff --git a/scripts/tests/twitter/integration/test_archive_graph.py b/scripts/tests/twitter/integration/test_archive_graph.py index 739ee49c..43e8bd51 100644 --- a/scripts/tests/twitter/integration/test_archive_graph.py +++ b/scripts/tests/twitter/integration/test_archive_graph.py @@ -17,21 +17,21 @@ import sqlmodel import twikit -from app.business.info_base.resolver import ResolverManager -from app.business.info_base.storage import HTTPStorage -from app.business.lexical_retrieval import LexicalRetrievalManager -from app.business.source import SourceManager -from app.persistence.info_base.repository import BlockRepository, RelationRepository -from app.persistence.info_base.uow import graph_uow -from app.schemas.info_base.block import BlockForm, BlockModel -from app.schemas.info_base.relation import RelationModel -from app.schemas.info_base.storage import StorageBlobModel -from app.schemas.job import JobModel -from app.schemas.lexical_retrieval import ( +from inkcre_core.business.info_base.resolver import ResolverManager +from inkcre_core.business.info_base.storage import HTTPStorage +from inkcre_core.business.lexical_retrieval import LexicalRetrievalManager +from inkcre_core.business.source import SourceManager +from inkcre_core.persistence.info_base.repository import BlockRepository, RelationRepository +from inkcre_core.persistence.info_base.uow import graph_uow +from inkcre_core.schemas.info_base.block import BlockForm, BlockModel +from inkcre_core.schemas.info_base.relation import RelationModel +from inkcre_core.schemas.info_base.storage import StorageBlobModel +from inkcre_core.schemas.job import JobModel +from inkcre_core.schemas.lexical_retrieval import ( BlockLexicalRecordModel, LexicalMaintenanceOptions, ) -from app.schemas.source import SourceModel +from inkcre_core.schemas.source import SourceModel from extensions.twitter.api import ( TwitterAPI, TwitterAPIResult, diff --git a/scripts/tests/twitter/integration/test_archive_http_roundtrip.py b/scripts/tests/twitter/integration/test_archive_http_roundtrip.py index ea6cb89a..e30a9b75 100644 --- a/scripts/tests/twitter/integration/test_archive_http_roundtrip.py +++ b/scripts/tests/twitter/integration/test_archive_http_roundtrip.py @@ -13,11 +13,11 @@ import sqlalchemy import sqlmodel -from app.persistence.info_base.uow import graph_uow -from app.business.peer import PeerManager -from app.business.source import SourceManager -from app.schemas.info_base.block import BlockModel -from app.schemas.info_base.relation import RelationModel +from inkcre_core.persistence.info_base.uow import graph_uow +from inkcre_core.business.peer import PeerManager +from inkcre_core.business.source import SourceManager +from inkcre_core.schemas.info_base.block import BlockModel +from inkcre_core.schemas.info_base.relation import RelationModel from extensions.twitter.archive_session import SESSION_COOKIE_NAME, _issue_session_token from extensions.twitter.reconcile import MEDIA_RESOLVER, POST_RESOLVER, USER_RESOLVER from tests.database import TestSession diff --git a/scripts/tests/twitter/test_archive_http.py b/scripts/tests/twitter/test_archive_http.py index 71a846ce..c0703579 100644 --- a/scripts/tests/twitter/test_archive_http.py +++ b/scripts/tests/twitter/test_archive_http.py @@ -10,8 +10,8 @@ import jwt import pytest -from app.business.peer import PeerManager -from app.database_contract.constants import JWT_ALGORITHM +from inkcre_core.business.peer import PeerManager +from inkcre_core.database_contract.constants import JWT_ALGORITHM from app.middleware import create_peer_jwt from app.settings import Settings, settings from extensions.twitter import archive_http, archive_session @@ -303,7 +303,7 @@ def test_session_does_not_renew_for_no_cors_subresource(archive_client): def test_extension_publishes_claimed_archive_routes(monkeypatch): from extensions.twitter import Extension - from app.business.source import SourceManager + from inkcre_core.business.source import SourceManager async def sync_source_types(): return None diff --git a/scripts/verify_postgrest_contract.py b/scripts/verify_postgrest_contract.py index d42e3418..c15eb359 100644 --- a/scripts/verify_postgrest_contract.py +++ b/scripts/verify_postgrest_contract.py @@ -14,7 +14,7 @@ PROJECT_ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(PROJECT_ROOT)) -from app.database_contract.constants import ( +from inkcre_core.database_contract.constants import ( JWT_ALGORITHM, JWT_AUDIENCE, JWT_ISSUER, diff --git a/tests/agent/integration/test_agent_definition.py b/tests/agent/integration/test_agent_definition.py index c7406e56..04772359 100644 --- a/tests/agent/integration/test_agent_definition.py +++ b/tests/agent/integration/test_agent_definition.py @@ -9,15 +9,15 @@ import sqlalchemy import sqlalchemy.exc -from app.business.agent import ( +from inkcre_core.business.agent import ( AgentManager, InMemoryThreadPersistenceBackend, TurnTermination, ) -from app.business.ai import AIManager +from inkcre_core.business.ai import AIManager from tests.database import TestSession -from app.schemas import AgentDefinitionModel -from app.schemas.ai import ( +from inkcre_core.schemas import AgentDefinitionModel +from inkcre_core.schemas.ai import ( AIModelModel, AIProviderModel, AssistantMessage, diff --git a/tests/agent/test_debug_trace.py b/tests/agent/test_debug_trace.py index 1136d071..ab87c3b4 100644 --- a/tests/agent/test_debug_trace.py +++ b/tests/agent/test_debug_trace.py @@ -7,15 +7,15 @@ import pydantic import pytest -from app.business.agent import ( +from inkcre_core.business.agent import ( BoundAgentTool, InMemoryThreadPersistenceBackend, Thread, ThreadState, TurnTermination, ) -from app.business.ai import AIManager -from app.schemas.ai import ( +from inkcre_core.business.ai import AIManager +from inkcre_core.schemas.ai import ( AssistantMessage, FunctionTool, SystemMessage, @@ -23,8 +23,9 @@ UserMessage, TextContentPart, ) -from app.settings import settings -from libs.obsrv.log_record import TRACE_ID +from dataclasses import replace +from inkcre_core import current_core +from inkcre_core.libs.obsrv.log_record import TRACE_ID class _Input(pydantic.BaseModel): @@ -33,7 +34,9 @@ class _Input(pydantic.BaseModel): @pytest.mark.parametrize("budget", [1, 2]) def test_trace_retains_tool_errors_and_actual_budget_outcome(monkeypatch, caplog, budget): - monkeypatch.setattr(settings.obsrv, "agent_debug", True) + monkeypatch.setattr( + current_core(), "options", replace(current_core().options, agent_debug=True) + ) caplog.set_level(logging.INFO, logger="inkcre") invoked = [] @@ -105,7 +108,9 @@ async def run(): def test_trace_records_cancellation_without_turn_recovery(monkeypatch, caplog): - monkeypatch.setattr(settings.obsrv, "agent_debug", True) + monkeypatch.setattr( + current_core(), "options", replace(current_core().options, agent_debug=True) + ) caplog.set_level(logging.INFO, logger="inkcre") async def run(): @@ -138,14 +143,18 @@ async def chat(cls, *args): def test_debug_disabled_and_broken_destination_do_not_raise(monkeypatch): - from app.business.agent.debug import trace + from inkcre_core.business.agent.debug import trace import uuid - monkeypatch.setattr(settings.obsrv, "agent_debug", False) + monkeypatch.setattr( + current_core(), "options", replace(current_core().options, agent_debug=False) + ) asyncio.run(trace("example", uuid.uuid4(), unsupported=object())) - monkeypatch.setattr(settings.obsrv, "agent_debug", True) + monkeypatch.setattr( + current_core(), "options", replace(current_core().options, agent_debug=True) + ) asyncio.run(trace("example", uuid.uuid4(), unsupported=object())) - from libs.obsrv.main import get_logger + from inkcre_core.libs.obsrv.main import get_logger def broken_destination(*args, **kwargs): raise OSError("destination unavailable") diff --git a/tests/ai/integration/test_ai_registry.py b/tests/ai/integration/test_ai_registry.py index dc021d88..a8095739 100644 --- a/tests/ai/integration/test_ai_registry.py +++ b/tests/ai/integration/test_ai_registry.py @@ -8,9 +8,9 @@ import sqlalchemy import sqlalchemy.exc -from app.business.ai import AIManager +from inkcre_core.business.ai import AIManager from tests.database import TestSession -from app.schemas.ai import ( +from inkcre_core.schemas.ai import ( AIModelModel, AIProviderModel, ChatCapability, diff --git a/tests/conftest.py b/tests/conftest.py index 129244d7..0d88762f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -49,3 +49,11 @@ def async_runner(): yield runner finally: runner.run(ASYNC_DB_ENGINE.dispose()) + + +@pytest.fixture(autouse=True) +def application_core_scope(): + from app.core import CORE + + with CORE.bind(): + yield CORE diff --git a/tests/database.py b/tests/database.py index 3c8b8367..a4ab9416 100644 --- a/tests/database.py +++ b/tests/database.py @@ -12,8 +12,8 @@ from sqlalchemy.pool import NullPool from app.settings import settings -from app.schemas.info_base.block import BlockForm, BlockModel -from app.schemas.info_base.relation import RelationModel +from inkcre_core.schemas.info_base.block import BlockForm, BlockModel +from inkcre_core.schemas.info_base.relation import RelationModel TEST_ENGINE = sqlmodel.create_engine(settings.database_url, poolclass=NullPool) diff --git a/tests/deployment_config/integration/test_config_resource.py b/tests/deployment_config/integration/test_config_resource.py index 5163521d..18f4cce1 100644 --- a/tests/deployment_config/integration/test_config_resource.py +++ b/tests/deployment_config/integration/test_config_resource.py @@ -1,6 +1,6 @@ """Black-box proof for shared config persistence and row timestamp ownership.""" -from app.business.deployment_config import DeploymentConfigService +from inkcre_core.business.deployment_config import DeploymentConfigService import os @@ -16,12 +16,12 @@ import sqlalchemy import sqlmodel -from app.business.deployment_config import DeploymentConfigManager +from inkcre_core.business.deployment_config import DeploymentConfigManager from app.engine import ASYNC_DB_ENGINE from tests.database import TestSession from app.routes.deployment_config import ROUTER -from app.schemas.info_base.block import BlockModel -from app.schemas.info_base.relation import RelationModel +from inkcre_core.schemas.info_base.block import BlockModel +from inkcre_core.schemas.info_base.relation import RelationModel pytestmark = pytest.mark.skipif( diff --git a/tests/extensions/local_files/test_reader.py b/tests/extensions/local_files/test_reader.py index 2f9d5c87..1a399a5b 100644 --- a/tests/extensions/local_files/test_reader.py +++ b/tests/extensions/local_files/test_reader.py @@ -9,7 +9,7 @@ from extensions.local_files.schema import FileMetadata, file_location from extensions.local_files.resolver import LocalFileResolver -from app.business.info_base.services import BlockService +from inkcre_core.business.info_base.services import BlockService def test_location_retains_machine_path_without_remote_file_access(): @@ -80,8 +80,8 @@ def test_failed_current_file_keeps_successful_body_readable(monkeypatch, async_r ) def test_v2_uses_the_body_format(monkeypatch, async_runner, resolver, content, expected): from extensions.local_files.resolver import LocalFileV2Resolver - from app.business.info_base.resolver import register_core_resolvers - from app.schemas.info_base.block import BlockModel + from inkcre_core.business.info_base.resolver import register_core_resolvers + from inkcre_core.schemas.info_base.block import BlockModel register_core_resolvers() now = datetime.datetime.now(datetime.UTC) diff --git a/tests/extensions/mail/acceptance/test_mail_vertical.py b/tests/extensions/mail/acceptance/test_mail_vertical.py index 3ec3f406..d639d8ad 100644 --- a/tests/extensions/mail/acceptance/test_mail_vertical.py +++ b/tests/extensions/mail/acceptance/test_mail_vertical.py @@ -1,6 +1,7 @@ """J1–J3: real IMAP -> Job -> graph -> Resolver/Storage acceptance.""" from __future__ import annotations +from inkcre_core import current_core import collections.abc import datetime @@ -12,23 +13,23 @@ import sqlalchemy import sqlmodel -from app.business.info_base.resolver import ResolverManager, register_core_resolvers -from app.business.info_base.storage import StorageManager -from app.business.job import JobManager -from app.business.lexical_retrieval import LexicalRetrievalManager -from app.business.source import ( +from inkcre_core.business.info_base.resolver import ResolverManager, register_core_resolvers +from inkcre_core.business.info_base.storage import StorageManager +from inkcre_core.business.job import JobManager +from inkcre_core.business.lexical_retrieval import LexicalRetrievalManager +from inkcre_core.business.source import ( SOURCE_BACKFILL_JOB_TYPE, SOURCE_COLLECT_JOB_TYPE, SourceManager, ) from tests.database import TestSession -from app.schemas.extension import ExtensionModel -from app.schemas.info_base.block import BlockModel -from app.schemas.info_base.relation import RelationModel -from app.schemas.info_base.storage import StorageBlobModel -from app.schemas.job import JobModel, JobStatus -from app.schemas.lexical_retrieval import LexicalMaintenanceOptions -from app.schemas.source import SourceModel +from inkcre_core.schemas.extension import ExtensionModel +from inkcre_core.schemas.info_base.block import BlockModel +from inkcre_core.schemas.info_base.relation import RelationModel +from inkcre_core.schemas.info_base.storage import StorageBlobModel +from inkcre_core.schemas.job import JobModel, JobStatus +from inkcre_core.schemas.lexical_retrieval import LexicalMaintenanceOptions +from inkcre_core.schemas.source import SourceModel from extensions.mail import Extension from extensions.mail.reconcile import ( EMAIL_ADDRESS_RESOLVER, @@ -102,7 +103,7 @@ def _reset_acceptance_database() -> None: } db.add(extension) db.commit() - SourceManager.SOURCES.clear() + current_core().sources.clear() async def _bootstrap_runtime_catalogs() -> None: diff --git a/tests/extensions/mail/test_newsletter.py b/tests/extensions/mail/test_newsletter.py index a82fea56..a9d27992 100644 --- a/tests/extensions/mail/test_newsletter.py +++ b/tests/extensions/mail/test_newsletter.py @@ -6,8 +6,8 @@ import pytest from bs4 import BeautifulSoup -from app.business.info_base.resolver import ResolverManager -from app.schemas.info_base.block import BlockModel +from inkcre_core.business.info_base.resolver import ResolverManager +from inkcre_core.schemas.info_base.block import BlockModel from extensions.mail.adapter import IMAPAdapter, _mailing_list, _walk_body_structure from extensions.mail.newsletter import ( NEWSLETTER_HTML_RESOLVER, diff --git a/tests/extensions/mail/test_newsletter_graph.py b/tests/extensions/mail/test_newsletter_graph.py index baf5fc89..9c0a57a9 100644 --- a/tests/extensions/mail/test_newsletter_graph.py +++ b/tests/extensions/mail/test_newsletter_graph.py @@ -5,10 +5,10 @@ import pytest -from app.business.info_base.resolver import ResolverManager -from app.business.source import SourceManager -from app.schemas.info_base.block import BlockForm -from app.persistence.source.uow import source_uow +from inkcre_core.business.info_base.resolver import ResolverManager +from inkcre_core.business.source import SourceManager +from inkcre_core.schemas.info_base.block import BlockForm +from inkcre_core.persistence.source.uow import source_uow from extensions.mail.newsletter import MAILING_LIST_RESOLVER, NEWSLETTER_HTML_RESOLVER from extensions.mail.reconcile import MailGraphReconciler from extensions.mail.schema import ( diff --git a/tests/extensions/memos/integration/test_postgresql_attachment_graph.py b/tests/extensions/memos/integration/test_postgresql_attachment_graph.py index 9d5a4fc6..ec9ba1a9 100644 --- a/tests/extensions/memos/integration/test_postgresql_attachment_graph.py +++ b/tests/extensions/memos/integration/test_postgresql_attachment_graph.py @@ -11,12 +11,12 @@ import sqlalchemy import sqlmodel -from app.business.info_base.storage import StorageManager -from app.business.info_base.storage.postgresql import PostgreSQLBlobPointer +from inkcre_core.business.info_base.storage import StorageManager +from inkcre_core.business.info_base.storage.postgresql import PostgreSQLBlobPointer from tests.database import TestSession -from app.schemas.info_base.block import BlockModel -from app.schemas.info_base.relation import RelationModel -from app.schemas.info_base.storage import StorageBlobModel +from inkcre_core.schemas.info_base.block import BlockModel +from inkcre_core.schemas.info_base.relation import RelationModel +from inkcre_core.schemas.info_base.storage import StorageBlobModel from extensions.memos import Extension from extensions.memos.family import ( AttachmentApplicationService, @@ -107,7 +107,7 @@ def memo_client(): def test_saved_pat_replacement_and_revocation_apply_without_restart(memo_client): """An admitted Peer can edit config without invoking the running Host.""" - from app.schemas.extension import ExtensionModel + from inkcre_core.schemas.extension import ExtensionModel old_token = "memos_pat_" + "A" * 32 new_token = "memos_pat_" + "B" * 32 @@ -135,9 +135,9 @@ def save(token: str | None): def test_connection_address_preserves_public_prefix_and_requires_peer_auth(memo_client): - from app.business.peer import PeerManager + from inkcre_core.business.peer import PeerManager from app.middleware import create_peer_jwt - from app.schemas.peer import PeerModel + from inkcre_core.schemas.peer import PeerModel from app.settings import settings client = memo_client diff --git a/tests/extensions/rss/integration/test_feed_collection.py b/tests/extensions/rss/integration/test_feed_collection.py index 423941e6..3e656718 100644 --- a/tests/extensions/rss/integration/test_feed_collection.py +++ b/tests/extensions/rss/integration/test_feed_collection.py @@ -1,6 +1,7 @@ """Real RSS/Atom HTTP doubles collected through PostgreSQL source jobs.""" from __future__ import annotations +from inkcre_core import current_core from tests.database import read_block, read_relations @@ -17,17 +18,17 @@ import pydantic import sqlmodel -from app.business.info_base.resolver import ResolverManager, register_core_resolvers -from app.business.info_base.storage import StorageManager -from app.business.info_base.storage.postgresql import PostgreSQLBlobPointer -from app.business.job import JobManager -from app.business.source import SOURCE_COLLECT_JOB_TYPE, SourceManager +from inkcre_core.business.info_base.resolver import ResolverManager, register_core_resolvers +from inkcre_core.business.info_base.storage import StorageManager +from inkcre_core.business.info_base.storage.postgresql import PostgreSQLBlobPointer +from inkcre_core.business.job import JobManager +from inkcre_core.business.source import SOURCE_COLLECT_JOB_TYPE, SourceManager from tests.database import TestSession -from app.schemas.info_base.block import BlockModel -from app.schemas.info_base.relation import RelationModel -from app.schemas.info_base.storage import StorageBlobModel -from app.schemas.job import JobModel, JobStatus -from app.schemas.source import SourceModel +from inkcre_core.schemas.info_base.block import BlockModel +from inkcre_core.schemas.info_base.relation import RelationModel +from inkcre_core.schemas.info_base.storage import StorageBlobModel +from inkcre_core.schemas.job import JobModel, JobStatus +from inkcre_core.schemas.source import SourceModel from extensions.rss import Extension from extensions.rss.api import register_api from extensions.rss.reconcile import ( @@ -324,7 +325,7 @@ def _cleanup(source_ids: set[int]) -> None: db_session.delete(blob) db_session.commit() for source_id in source_ids: - SourceManager.SOURCES.pop(source_id, None) + current_core().sources.pop(source_id, None) async def _exercise_rss() -> None: diff --git a/tests/extensions/runtime_support.py b/tests/extensions/runtime_support.py index 28a09b3b..38a1b47c 100644 --- a/tests/extensions/runtime_support.py +++ b/tests/extensions/runtime_support.py @@ -56,14 +56,17 @@ def publish_extension( raise_server_exceptions: bool = True, persist_config: bool = False, ) -> PublishedExtension: + from app.core import CORE, CoreMiddleware + runtime_app = app or fastapi.FastAPI() + runtime_app.add_middleware(CoreMiddleware, core=CORE) runtime_config = dict(config or {}) runtime_state: dict[str, typing.Any] = {} if persist_config: if not os.getenv("INKCRE_TEST_DATABASE_URL"): raise RuntimeError("Persisted Extension setup requires an explicit test database") - from app.schemas.extension import ExtensionModel + from inkcre_core.schemas.extension import ExtensionModel from tests.database import TestSession with TestSession() as session: diff --git a/tests/info_base/integration/test_graph_commands.py b/tests/info_base/integration/test_graph_commands.py index 1737d229..18d41a2d 100644 --- a/tests/info_base/integration/test_graph_commands.py +++ b/tests/info_base/integration/test_graph_commands.py @@ -10,12 +10,16 @@ import psycopg.errors import sqlmodel -from app.business.info_base.commands import persist_graph, submit_graph, submit_stars -from app.persistence.info_base.uow import graph_uow -from app.business.info_base.resolver import register_core_resolvers +from inkcre_core.business.info_base.commands import ( + persist_graph, + submit_graph, + submit_stars, +) +from inkcre_core.persistence.info_base.uow import graph_uow +from inkcre_core.business.info_base.resolver import register_core_resolvers from tests.database import TestSession -from app.schemas.info_base.block import BlockForm, BlockModel -from app.schemas.info_base.main import ( +from inkcre_core.schemas.info_base.block import BlockForm, BlockModel +from inkcre_core.schemas.info_base.main import ( GraphBlockForm, GraphForm, GraphRelationForm, @@ -23,7 +27,7 @@ OutArcForm, StarsGraphForm, ) -from app.schemas.info_base.relation import RelationForm, RelationModel +from inkcre_core.schemas.info_base.relation import RelationForm, RelationModel pytestmark = pytest.mark.skipif( @@ -228,9 +232,9 @@ async def cancel_then_retry(): def test_blob_graph_transaction_and_hydration(async_runner): - from app.business.info_base.storage import StorageManager, WritableStorage - from app.business.info_base.services import BlockService - from app.schemas.info_base.storage import StorageBlobModel + from inkcre_core.business.info_base.storage import StorageManager, WritableStorage + from inkcre_core.business.info_base.services import BlockService + from inkcre_core.schemas.info_base.storage import StorageBlobModel async def scenario(): await StorageManager.setup_builtin_storages_async() @@ -256,7 +260,7 @@ async def scenario(): async with graph_uow() as uow: rolled_back_pointer = await storage.create_content(b"not committed", uow.storage) raise RuntimeError("abort graph") - from app.business.info_base.storage.postgresql import PostgreSQLBlobPointer + from inkcre_core.business.info_base.storage.postgresql import PostgreSQLBlobPointer blob_id = PostgreSQLBlobPointer.model_validate_json(rolled_back_pointer).blob_id with TestSession() as db: @@ -274,7 +278,7 @@ async def scenario(): def test_github_stars_retain_node_identity_and_reject_ambiguity(async_runner): from extensions.github.resolver import GitHubAccountResolver, GitHubGraphIntegrityError from extensions.github.schema import GitHubAccount - from app.business.info_base.commands import persist_stars + from inkcre_core.business.info_base.commands import persist_stars account = GitHubAccount( node_id=uuid.uuid4().hex, kind="user", login="before", url="https://github.com/before" diff --git a/tests/info_base/test_markdown.py b/tests/info_base/test_markdown.py index 34926663..f5569728 100644 --- a/tests/info_base/test_markdown.py +++ b/tests/info_base/test_markdown.py @@ -3,12 +3,12 @@ import datetime from unittest.mock import AsyncMock -from app.business.info_base.resolver import ( +from inkcre_core.business.info_base.resolver import ( MarkdownResolver, ResolverManager, register_core_resolvers, ) -from app.schemas.info_base.block import BlockModel +from inkcre_core.schemas.info_base.block import BlockModel def test_markdown_keeps_source_and_unicode(async_runner, monkeypatch): diff --git a/tests/lexical_retrieval/acceptance/test_multimodal_live.py b/tests/lexical_retrieval/acceptance/test_multimodal_live.py index bb69e69e..afb1960a 100644 --- a/tests/lexical_retrieval/acceptance/test_multimodal_live.py +++ b/tests/lexical_retrieval/acceptance/test_multimodal_live.py @@ -1,8 +1,8 @@ """Credentialed real-provider media -> graph -> lexical acceptance.""" -from app.business.info_base.commands import get_related_block -from app.business.deployment_config import DeploymentConfigService -from app.persistence.info_base.uow import graph_uow +from inkcre_core.business.info_base.commands import get_related_block +from inkcre_core.business.deployment_config import DeploymentConfigService +from inkcre_core.persistence.info_base.uow import graph_uow import os @@ -12,34 +12,34 @@ import pytest import sqlalchemy -from app.business.ai import AIManager -from app.business.info_base.resolver import ResolverManager, register_core_resolvers -from app.business.info_base.resolver.audio import ( +from inkcre_core.business.ai import AIManager +from inkcre_core.business.info_base.resolver import ResolverManager, register_core_resolvers +from inkcre_core.business.info_base.resolver.audio import ( AUDIO_RESOLVER_CONFIG_KEY, AUDIO_RESOLVER_CONFIG_SCHEMA, ) -from app.business.info_base.resolver.image import ( +from inkcre_core.business.info_base.resolver.image import ( IMAGE_RESOLVER_CONFIG_KEY, IMAGE_RESOLVER_CONFIG_SCHEMA, ) -from app.business.info_base.resolver.video import ( +from inkcre_core.business.info_base.resolver.video import ( VIDEO_RESOLVER_CONFIG_KEY, VIDEO_RESOLVER_CONFIG_SCHEMA, ) -from app.business.info_base.storage import StorageManager, WritableStorage -from app.business.lexical_retrieval import LexicalRetrievalManager -from app.business.organization import SUBMIT_GRAPH_TOOL -from app.business.organization_media import ( +from inkcre_core.business.info_base.storage import StorageManager, WritableStorage +from inkcre_core.business.lexical_retrieval import LexicalRetrievalManager +from inkcre_core.business.organization import SUBMIT_GRAPH_TOOL +from inkcre_core.business.organization_media import ( MEDIA_INTERPRETATION_CONFIG_KEY, MEDIA_INTERPRETATION_CONFIG_SCHEMA, interpret_missing_media, ) from tests.database import TestSession -from app.schemas import AgentDefinitionModel -from app.schemas.ai import AIModelModel, AIProviderModel, ChatCapability -from app.schemas.deployment_config import DeploymentConfigModel -from app.schemas.info_base.block import BlockForm, BlockModel -from app.schemas.lexical_retrieval import LexicalMaintenanceOptions +from inkcre_core.schemas import AgentDefinitionModel +from inkcre_core.schemas.ai import AIModelModel, AIProviderModel, ChatCapability +from inkcre_core.schemas.deployment_config import DeploymentConfigModel +from inkcre_core.schemas.info_base.block import BlockForm, BlockModel +from inkcre_core.schemas.lexical_retrieval import LexicalMaintenanceOptions from .prepare_media_assets import prepare_assets diff --git a/tests/lexical_retrieval/integration/test_lexical_retrieval.py b/tests/lexical_retrieval/integration/test_lexical_retrieval.py index b751b263..6c7f7ac5 100644 --- a/tests/lexical_retrieval/integration/test_lexical_retrieval.py +++ b/tests/lexical_retrieval/integration/test_lexical_retrieval.py @@ -1,6 +1,6 @@ """Real PostgreSQL proof for lexical projection, freshness, and ranking.""" -from app.business.info_base.services import BlockService +from inkcre_core.business.info_base.services import BlockService from tests.database import seed_block @@ -9,19 +9,19 @@ import pytest import sqlalchemy -from app.business.info_base.resolver import register_core_resolvers -from app.business.cron import CronManager -from app.business.job import JobManager -from app.business.lexical_retrieval import ( +from inkcre_core.business.info_base.resolver import register_core_resolvers +from inkcre_core.business.cron import CronManager +from inkcre_core.business.job import JobManager +from inkcre_core.business.lexical_retrieval import ( LEXICAL_MAINTAIN_JOB_TYPE, LEXICAL_REBUILD_JOB_TYPE, LexicalRetrievalManager, ) from tests.database import TestSession -from app.schemas.info_base.block import BlockForm -from app.schemas.cron import CronModel -from app.schemas.job import JobModel, JobStatus -from app.schemas.lexical_retrieval import LexicalMaintenanceOptions +from inkcre_core.schemas.info_base.block import BlockForm +from inkcre_core.schemas.cron import CronModel +from inkcre_core.schemas.job import JobModel, JobStatus +from inkcre_core.schemas.lexical_retrieval import LexicalMaintenanceOptions pytestmark = pytest.mark.skipif( @@ -30,6 +30,15 @@ ) +@pytest.fixture +def job_execution(async_runner): + JobManager.start() + try: + yield + finally: + async_runner.run(JobManager.shutdown()) + + def _reset() -> None: with TestSession() as db: db.connection().execute( @@ -123,6 +132,7 @@ def test_unknown_resolver_is_bounded_unavailable_diagnostic( def test_direct_and_cron_created_jobs_share_exact_handler_path( async_runner, + job_execution, ): _reset() register_core_resolvers() diff --git a/tests/migrations/test_extension_peer_enabled_rpc.py b/tests/migrations/test_extension_peer_enabled_rpc.py index 19660330..3f37d3c3 100644 --- a/tests/migrations/test_extension_peer_enabled_rpc.py +++ b/tests/migrations/test_extension_peer_enabled_rpc.py @@ -19,15 +19,15 @@ import sqlmodel from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine -from app.business.cron import CronManager +from inkcre_core.business.cron import CronManager from app.business.extension.errors import ExtensionStateConflictError from app.business.extension.state import ExtensionStateService -from app.business.job import JobManager -from app.business.source import SOURCE_COLLECT_JOB_TYPE, SourceManager -from app.persistence.extension import uow as extension_uow -from app.schemas.extension import ExtensionModel, ExtensionPeerParticipationModel -from app.schemas.peer import PeerModel -from app.schemas.cron import CronForm +from inkcre_core.business.job import JobManager +from inkcre_core.business.source import SOURCE_COLLECT_JOB_TYPE, SourceManager +from inkcre_core.persistence.extension import uow as extension_uow +from inkcre_core.schemas.extension import ExtensionModel, ExtensionPeerParticipationModel +from inkcre_core.schemas.peer import PeerModel +from inkcre_core.schemas.cron import CronForm from extensions.twitter.bookmark import Source as BookmarkSource @@ -462,9 +462,9 @@ def test_setup_source_and_cron_use_simple_core_owned_operations( ): import asyncio from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine - from app.persistence.cron import uow as cron_uow - from app.persistence.job import uow as job_uow - from app.persistence.source import uow as source_uow + from inkcre_core.persistence.cron import uow as cron_uow + from inkcre_core.persistence.job import uow as job_uow + from inkcre_core.persistence.source import uow as source_uow engine = create_async_engine(setup_domain_engine.url) factory = async_sessionmaker(engine, expire_on_commit=False) diff --git a/tests/migrations/test_memos_attachment_v2_migration.py b/tests/migrations/test_memos_attachment_v2_migration.py index 059a76e2..a2e267b8 100644 --- a/tests/migrations/test_memos_attachment_v2_migration.py +++ b/tests/migrations/test_memos_attachment_v2_migration.py @@ -17,15 +17,16 @@ import pytest import sqlalchemy import sqlmodel -from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine +from sqlalchemy.ext.asyncio import create_async_engine from sqlalchemy.pool import NullPool from app.database_contract.connection import normalized_database_url, psycopg_database_url -from app.business.info_base.storage.postgresql import PostgreSQLBlobPointer +from inkcre_core import Core +from inkcre_core.business.info_base.storage.postgresql import PostgreSQLBlobPointer from tests.database import TestSession -from app.schemas.info_base.block import BlockModel -from app.schemas.info_base.relation import RelationModel -from app.schemas.info_base.storage import StorageBlobModel +from inkcre_core.schemas.info_base.block import BlockModel +from inkcre_core.schemas.info_base.relation import RelationModel +from inkcre_core.schemas.info_base.storage import StorageBlobModel from extensions.memos import Extension from extensions.memos.family import AttachmentApplicationService, CanonicalAttachment @@ -54,6 +55,7 @@ def attachment_database(monkeypatch): ) engine = sqlalchemy.create_engine(database_url, poolclass=NullPool) async_engine = create_async_engine(database_url, poolclass=NullPool) + core = Core(async_engine) config = Config(PROJECT_ROOT / "alembic.ini") config.set_main_option("script_location", str(PROJECT_ROOT / "migrations")) with psycopg.connect(psycopg_database_url(owner_url), autocommit=True) as owner: @@ -61,10 +63,6 @@ def attachment_database(monkeypatch): try: monkeypatch.setenv("MIGRATION_DATABASE_URL", database_url) monkeypatch.setattr("tests.database.TEST_ENGINE", engine) - monkeypatch.setattr( - "app.persistence.info_base.uow.AsyncSessionFactory", - async_sessionmaker(async_engine, expire_on_commit=False), - ) command.upgrade(config, ATTACHMENT_REVISION) with engine.begin() as connection: connection.execute( @@ -79,8 +77,10 @@ def attachment_database(monkeypatch): "VALUES (-4, 'postgresql_binary', '{}'::jsonb)" ) ) - yield config + with core.bind(): + yield config finally: + asyncio.run(core.aclose()) engine.dispose() asyncio.run(async_engine.dispose()) with psycopg.connect(psycopg_database_url(owner_url), autocommit=True) as owner: diff --git a/tests/organization/acceptance/test_black_box.py b/tests/organization/acceptance/test_black_box.py index 19f8b303..67cbc495 100644 --- a/tests/organization/acceptance/test_black_box.py +++ b/tests/organization/acceptance/test_black_box.py @@ -2,7 +2,7 @@ from __future__ import annotations -from app.business.deployment_config import DeploymentConfigService +from inkcre_core.business.deployment_config import DeploymentConfigService from tests.database import seed_block, seed_relation @@ -15,19 +15,19 @@ import sqlalchemy import sqlmodel -from app.business.ai import AIManager -from app.business.graph_navigation_retrieval import GraphNavigationRetrievalManager -from app.business.graph_navigation_retrieval.tools import ( +from inkcre_core.business.ai import AIManager +from inkcre_core.business.graph_navigation_retrieval import GraphNavigationRetrievalManager +from inkcre_core.business.graph_navigation_retrieval.tools import ( FIND_PATH_TOOL, GET_CONNECTED_COMPONENTS_TOOL, GET_ENTITY_NEIGHBORHOOD_TOOL, ) -from app.business.info_base.tools import GET_ENTITIES_TOOL, RETRIEVE_TOOL -from app.business.info_base.resolver import ResolverManager, register_core_resolvers -from app.business.info_base.resolver.tools import RESOLVER_TOOL -from app.business.job import JobManager -from app.business.lexical_retrieval import LexicalRetrievalManager -from app.business.organization import ( +from inkcre_core.business.info_base.tools import GET_ENTITIES_TOOL, RETRIEVE_TOOL +from inkcre_core.business.info_base.resolver import ResolverManager, register_core_resolvers +from inkcre_core.business.info_base.resolver.tools import RESOLVER_TOOL +from inkcre_core.business.job import JobManager +from inkcre_core.business.lexical_retrieval import LexicalRetrievalManager +from inkcre_core.business.organization import ( ANCHOR_EXISTING_REFERENT_TOOL, CREATE_SYNTHESIS_TOOL, DRAFT_GRAPH_TOOL, @@ -39,16 +39,16 @@ SUBMIT_GRAPH_TOOL, register_core_organization_behaviors, ) -from app.business.organization.duplicate_assertion import ( +from inkcre_core.business.organization.duplicate_assertion import ( DUPLICATE_ASSERTION_CONFIG_KEY, DUPLICATE_ASSERTION_CONFIG_SCHEMA, DUPLICATES_ASSERTION_RELATION, ) -from app.business.organization.evidence_stance import ( +from inkcre_core.business.organization.evidence_stance import ( EVIDENCE_STANCE_CONFIG_KEY, EVIDENCE_STANCE_CONFIG_SCHEMA, ) -from app.business.organization.jobs import ( +from inkcre_core.business.organization.jobs import ( DUPLICATE_ASSERTION_JOB_TYPE, EVIDENCE_STANCE_JOB_TYPE, REFERENT_ANCHORING_JOB_TYPE, @@ -57,39 +57,42 @@ SUPERSESSION_JOB_TYPE, SYNTHESIS_JOB_TYPE, ) -from app.business.organization.referent_anchoring import ( +from inkcre_core.business.organization.referent_anchoring import ( HAS_MENTION_RELATION, REFERENT_ANCHORING_CONFIG_KEY, REFERENT_ANCHORING_CONFIG_SCHEMA, REFERS_TO_RELATION, ) -from app.business.organization.refinement import ( +from inkcre_core.business.organization.refinement import ( REFINEMENT_CONFIG_KEY, REFINEMENT_CONFIG_SCHEMA, ) -from app.business.organization.rumination import ( +from inkcre_core.business.organization.rumination import ( RUMINATION_CONFIG_KEY, RUMINATION_CONFIG_SCHEMA, ) -from app.business.organization.supersession import ( +from inkcre_core.business.organization.supersession import ( SUPERSESSION_CONFIG_KEY, SUPERSESSION_CONFIG_SCHEMA, SUPERSESSION_BEHAVIOR, SUPERSEDES_RELATION, ) -from app.business.organization.synthesis import ( +from inkcre_core.business.organization.synthesis import ( SYNTHESIS_CONFIG_KEY, SYNTHESIS_CONFIG_SCHEMA, SYNTHESIS_RELATION, ) from tests.database import TestSession -from app.schemas import AgentDefinitionModel -from app.schemas.ai import AIModelModel, AIProviderModel, ChatCapability -from app.schemas.deployment_config import DeploymentConfigModel, DeploymentConfigView -from app.schemas.info_base.block import BlockForm, BlockModel -from app.schemas.info_base.relation import RelationModel -from app.schemas.job import JobModel, JobStatus -from app.schemas.lexical_retrieval import LexicalMaintenanceOptions +from inkcre_core.schemas import AgentDefinitionModel +from inkcre_core.schemas.ai import AIModelModel, AIProviderModel, ChatCapability +from inkcre_core.schemas.deployment_config import ( + DeploymentConfigModel, + DeploymentConfigView, +) +from inkcre_core.schemas.info_base.block import BlockForm, BlockModel +from inkcre_core.schemas.info_base.relation import RelationModel +from inkcre_core.schemas.job import JobModel, JobStatus +from inkcre_core.schemas.lexical_retrieval import LexicalMaintenanceOptions from .corpus import CorpusManifest, load_manifest, read_artifact diff --git a/tests/organization/integration/test_behavior_graph.py b/tests/organization/integration/test_behavior_graph.py index c0f5cb1e..38f2d469 100644 --- a/tests/organization/integration/test_behavior_graph.py +++ b/tests/organization/integration/test_behavior_graph.py @@ -9,9 +9,9 @@ import pytest import sqlalchemy -from app.business.graph_navigation_retrieval import GraphNavigationRetrievalManager -from app.business.info_base.resolver import register_core_resolvers -from app.business.organization import ( +from inkcre_core.business.graph_navigation_retrieval import GraphNavigationRetrievalManager +from inkcre_core.business.info_base.resolver import register_core_resolvers +from inkcre_core.business.organization import ( DuplicateAssertionBehaviorResolver, EvidenceStanceBehaviorResolver, ExistingReferentAnchoringBehaviorResolver, @@ -21,7 +21,7 @@ register_core_organization_behaviors, ) from tests.database import TestSession -from app.schemas.info_base.block import BlockForm +from inkcre_core.schemas.info_base.block import BlockForm pytestmark = pytest.mark.skipif( @@ -222,8 +222,8 @@ def test_connected_component_reports_missing_and_bounded_incomplete_proof( def test_synthesis_rolls_back_block_and_basis_on_write_failure(async_runner, monkeypatch): - from app.persistence.info_base.repository import RelationRepository - from app.persistence.info_base.uow import graph_uow + from inkcre_core.persistence.info_base.repository import RelationRepository + from inkcre_core.persistence.info_base.uow import graph_uow marker = uuid.uuid4().hex diff --git a/tests/organization/integration/test_media_interpretation.py b/tests/organization/integration/test_media_interpretation.py index a0b0d999..7e538e89 100644 --- a/tests/organization/integration/test_media_interpretation.py +++ b/tests/organization/integration/test_media_interpretation.py @@ -1,7 +1,7 @@ """Real-storage vertical proof for media textualization and interpretation Jobs.""" -from app.business.deployment_config import DeploymentConfigService -from app.persistence.info_base.uow import graph_uow +from inkcre_core.business.deployment_config import DeploymentConfigService +from inkcre_core.persistence.info_base.uow import graph_uow from tests.database import read_relations @@ -13,34 +13,34 @@ import pytest import sqlalchemy -import app.business.organization_job # noqa: F401 # exact Job registration -from app.business.ai import AIManager -from app.business.cron import CronManager -from app.business.info_base.resolver import register_core_resolvers -from app.business.info_base.resolver.audio import ( +import inkcre_core.business.organization_job # noqa: F401 # exact Job registration +from inkcre_core.business.ai import AIManager +from inkcre_core.business.cron import CronManager +from inkcre_core.business.info_base.resolver import register_core_resolvers +from inkcre_core.business.info_base.resolver.audio import ( AUDIO_RESOLVER_CONFIG_KEY, AUDIO_RESOLVER_CONFIG_SCHEMA, ) -from app.business.info_base.resolver.image import ( +from inkcre_core.business.info_base.resolver.image import ( IMAGE_RESOLVER_CONFIG_KEY, IMAGE_RESOLVER_CONFIG_SCHEMA, ) -from app.business.info_base.resolver.video import ( +from inkcre_core.business.info_base.resolver.video import ( VIDEO_RESOLVER_CONFIG_KEY, VIDEO_RESOLVER_CONFIG_SCHEMA, ) -from app.business.info_base.storage import StorageManager, WritableStorage -from app.business.job import JobManager -from app.business.lexical_retrieval import LexicalRetrievalManager -from app.business.organization import SUBMIT_GRAPH_TOOL -from app.business.organization_media import ( +from inkcre_core.business.info_base.storage import StorageManager, WritableStorage +from inkcre_core.business.job import JobManager +from inkcre_core.business.lexical_retrieval import LexicalRetrievalManager +from inkcre_core.business.organization import SUBMIT_GRAPH_TOOL +from inkcre_core.business.organization_media import ( MEDIA_INTERPRETATION_CONFIG_KEY, MEDIA_INTERPRETATION_CONFIG_SCHEMA, MEDIA_INTERPRETATION_JOB_TYPE, ) from tests.database import TestSession -from app.schemas import AgentDefinitionModel -from app.schemas.ai import ( +from inkcre_core.schemas import AgentDefinitionModel +from inkcre_core.schemas.ai import ( AIModelModel, AIProviderModel, AssistantMessage, @@ -53,11 +53,11 @@ UserMessage, VideoContentPart, ) -from app.schemas.cron import CronModel -from app.schemas.deployment_config import DeploymentConfigModel -from app.schemas.info_base.block import BlockForm, BlockModel -from app.schemas.job import JobModel, JobStatus -from app.schemas.lexical_retrieval import LexicalMaintenanceOptions +from inkcre_core.schemas.cron import CronModel +from inkcre_core.schemas.deployment_config import DeploymentConfigModel +from inkcre_core.schemas.info_base.block import BlockForm, BlockModel +from inkcre_core.schemas.job import JobModel, JobStatus +from inkcre_core.schemas.lexical_retrieval import LexicalMaintenanceOptions pytestmark = pytest.mark.skipif( @@ -74,6 +74,15 @@ ) +@pytest.fixture +def job_execution(async_runner): + JobManager.start() + try: + yield + finally: + async_runner.run(JobManager.shutdown()) + + def _reset_runtime_rows() -> None: with TestSession() as db: db.connection().execute( @@ -133,6 +142,7 @@ def _related(block: int, role: str) -> tuple[BlockModel, ...]: def test_media_textualization_interpretation_and_lexical_recall( async_runner, + job_execution, monkeypatch: pytest.MonkeyPatch, semantic_content_assets: Path, ) -> None: diff --git a/tests/organization/integration/test_rumination_graph.py b/tests/organization/integration/test_rumination_graph.py index 6e3836d7..6368e509 100644 --- a/tests/organization/integration/test_rumination_graph.py +++ b/tests/organization/integration/test_rumination_graph.py @@ -1,6 +1,6 @@ """Real PostgreSQL context and draft-to-submit graph proof.""" -from app.business.deployment_config import DeploymentConfigService +from inkcre_core.business.deployment_config import DeploymentConfigService from tests.database import seed_block, seed_relation @@ -15,10 +15,10 @@ import sqlalchemy import sqlmodel -from app.business.agent import AgentManager -from app.business.ai import AIManager -from app.business.info_base.resolver import register_core_resolvers -from app.business.organization import ( +from inkcre_core.business.agent import AgentManager +from inkcre_core.business.ai import AIManager +from inkcre_core.business.info_base.resolver import register_core_resolvers +from inkcre_core.business.organization import ( DRAFT_GRAPH_TOOL, RUMINATION_CONFIG_KEY, RUMINATION_CONFIG_SCHEMA, @@ -26,8 +26,8 @@ RuminationBehaviorResolver, ) from tests.database import TestSession -from app.schemas import AgentDefinitionModel -from app.schemas.ai import ( +from inkcre_core.schemas import AgentDefinitionModel +from inkcre_core.schemas.ai import ( AIModelModel, AIProviderModel, AssistantMessage, @@ -37,9 +37,9 @@ ToolCall, ToolResultMessage, ) -from app.schemas.deployment_config import DeploymentConfigModel -from app.schemas.info_base.block import BlockForm, BlockModel -from app.schemas.info_base.relation import RelationModel +from inkcre_core.schemas.deployment_config import DeploymentConfigModel +from inkcre_core.schemas.info_base.block import BlockForm, BlockModel +from inkcre_core.schemas.info_base.relation import RelationModel pytestmark = pytest.mark.skipif( diff --git a/tests/peer/integration/test_peer_runtime.py b/tests/peer/integration/test_peer_runtime.py index d75659e2..11a02f8d 100644 --- a/tests/peer/integration/test_peer_runtime.py +++ b/tests/peer/integration/test_peer_runtime.py @@ -1,5 +1,7 @@ """Real PostgreSQL proof for Peer snapshots, database-time lease and discovery.""" +from dataclasses import replace +from inkcre_core import current_core import datetime import os import time @@ -8,9 +10,9 @@ import pytest import sqlalchemy -from app.business.peer import PeerHTTPInbound, PeerManager +from inkcre_core.business.peer import PeerHTTPInbound, PeerManager from tests.database import TestSession -from app.schemas.peer import PEER_HTTP_PROTOCOL, PeerModel +from inkcre_core.schemas.peer import PEER_HTTP_PROTOCOL, PeerModel from app.version import APPLICATION_VERSION @@ -23,19 +25,22 @@ def test_real_snapshot_database_lease_and_candidate_filtering(monkeypatch, async_runner): - original_peer = PeerManager.get_current_peer_ref() - original_inbounds = PeerManager._INBOUNDS - original_outbounds = PeerManager._OUTBOUNDS + original_options = current_core().options + original_inbounds = current_core().inbounds + original_outbounds = current_core().outbounds local = uuid.uuid4() live = uuid.uuid4() expired = uuid.uuid4() malformed = uuid.uuid4() peer_ids = (local, live, expired, malformed) try: - monkeypatch.setattr(PeerManager, "_INBOUNDS", {}) - monkeypatch.setattr(PeerManager, "_OUTBOUNDS", {}) - monkeypatch.setattr("app.business.peer.main.settings.peer_id", local) - monkeypatch.setattr("app.business.peer.main.settings.peer_name", "integration-local") + monkeypatch.setattr(current_core(), "inbounds", {}) + monkeypatch.setattr(current_core(), "outbounds", {}) + monkeypatch.setattr( + current_core(), + "options", + replace(current_core().options, peer_id=local, peer_name="integration-local"), + ) async_runner.run(PeerManager.register_self()) PeerManager.setup_builtin_outbounds() @@ -132,9 +137,9 @@ def test_real_snapshot_database_lease_and_candidate_filtering(monkeypatch, async assert cleared is not None assert cleared.lease_expires_at is None finally: - monkeypatch.setattr(PeerManager, "_INBOUNDS", original_inbounds) - monkeypatch.setattr(PeerManager, "_OUTBOUNDS", original_outbounds) - monkeypatch.setattr("app.business.peer.main.settings.peer_id", original_peer) + monkeypatch.setattr(current_core(), "inbounds", original_inbounds) + monkeypatch.setattr(current_core(), "outbounds", original_outbounds) + monkeypatch.setattr(current_core(), "options", original_options) with TestSession() as db: db.connection().execute( sqlalchemy.text("DELETE FROM inkcre.peers WHERE id = ANY(:ids)"), @@ -148,11 +153,21 @@ def test_failed_cold_restore_preserves_intent_and_reports_not_ready(monkeypatch) from fastapi.testclient import TestClient import run - from app.schemas.extension import ExtensionModel, ExtensionPeerParticipationModel + from inkcre_core.schemas.extension import ExtensionModel, ExtensionPeerParticipationModel + + from app.core import create_core peer = uuid.uuid4() name = f"issue121/missing-{peer.hex}" - monkeypatch.setattr("app.settings.settings.peer_id", peer) + # This lifespan closes its owned Core; do not close other tests' service instance. + service_core = create_core() + service_core.options = replace(service_core.options, peer_id=peer) + original_core = run.CORE + monkeypatch.setattr(run, "CORE", service_core) + monkeypatch.setattr(run.api_app, "middleware_stack", None) + for middleware in run.api_app.user_middleware: + if middleware.kwargs.get("core") is original_core: + monkeypatch.setitem(middleware.kwargs, "core", service_core) monkeypatch.setattr(run, "SKIP_EXTENSION_START", False) with TestSession() as db: db.add( diff --git a/tests/semantic_retrieval/acceptance/test_vertical_quality.py b/tests/semantic_retrieval/acceptance/test_vertical_quality.py index 1016e3bd..4204c000 100644 --- a/tests/semantic_retrieval/acceptance/test_vertical_quality.py +++ b/tests/semantic_retrieval/acceptance/test_vertical_quality.py @@ -1,9 +1,10 @@ """Black-box corpus ingestion and explicit real-provider semantic quality gates.""" from __future__ import annotations +from inkcre_core import current_core -from app.business.deployment_config import DeploymentConfigService -from app.business.info_base.commands import submit_stars +from inkcre_core.business.deployment_config import DeploymentConfigService +from inkcre_core.business.info_base.commands import submit_stars from tests.database import read_block, read_relations @@ -20,12 +21,12 @@ import sqlalchemy import sqlmodel -from app.business.ai import AIManager -from app.business.info_base.resolver import register_core_resolvers -from app.business.info_base.resolver.html import HTMLResolver -from app.business.info_base.storage import StorageManager -from app.business.info_base.storage.postgresql import PostgreSQLBlobPointer -from app.business.organization import ( +from inkcre_core.business.ai import AIManager +from inkcre_core.business.info_base.resolver import register_core_resolvers +from inkcre_core.business.info_base.resolver.html import HTMLResolver +from inkcre_core.business.info_base.storage import StorageManager +from inkcre_core.business.info_base.storage.postgresql import PostgreSQLBlobPointer +from inkcre_core.business.organization import ( DRAFT_GRAPH_TOOL, GET_DRAFT_GRAPH_SCHEMA_TOOL, RUMINATION_CONFIG_KEY, @@ -33,14 +34,14 @@ SUBMIT_GRAPH_TOOL, RuminationBehaviorResolver, ) -from app.business.semantic_retrieval import SemanticRetrievalManager -from app.business.job import JobManager -from app.business.lexical_retrieval import LexicalRetrievalManager -from app.business.source import SOURCE_COLLECT_JOB_TYPE, SourceManager +from inkcre_core.business.semantic_retrieval import SemanticRetrievalManager +from inkcre_core.business.job import JobManager +from inkcre_core.business.lexical_retrieval import LexicalRetrievalManager +from inkcre_core.business.source import SOURCE_COLLECT_JOB_TYPE, SourceManager from tests.database import TestSession -from app.schemas import AgentDefinitionModel +from inkcre_core.schemas import AgentDefinitionModel from tests.extensions.runtime_support import publish_extension -from app.schemas.ai import ( +from inkcre_core.schemas.ai import ( AIModelModel, AIProviderModel, AssistantMessage, @@ -53,16 +54,19 @@ ToolResultMessage, UserMessage, ) -from app.schemas.deployment_config import DeploymentConfigModel, DeploymentConfigView -from app.schemas.info_base.block import BlockModel -from app.schemas.info_base.relation import RelationModel -from app.schemas.semantic_retrieval import ( +from inkcre_core.schemas.deployment_config import ( + DeploymentConfigModel, + DeploymentConfigView, +) +from inkcre_core.schemas.info_base.block import BlockModel +from inkcre_core.schemas.info_base.relation import RelationModel +from inkcre_core.schemas.semantic_retrieval import ( EmbeddingMaintenanceOptions, VectorRetrievalOptions, ) -from app.schemas.job import JobModel, JobStatus -from app.schemas.lexical_retrieval import LexicalMaintenanceOptions -from app.schemas.source import SourceModel +from inkcre_core.schemas.job import JobModel, JobStatus +from inkcre_core.schemas.lexical_retrieval import LexicalMaintenanceOptions +from inkcre_core.schemas.source import SourceModel from extensions.memos import Extension as MemosExtension from extensions.rss import Extension as RSSExtension from extensions.rss.reconcile import ( @@ -458,7 +462,7 @@ def _cleanup_corpus(run: CorpusRun) -> None: ) db.commit() for source_id in run.source_ids: - SourceManager.SOURCES.pop(source_id, None) + current_core().sources.pop(source_id, None) async def _close_corpus(run: CorpusRun) -> None: diff --git a/tests/semantic_retrieval/integration/test_semantic_retrieval.py b/tests/semantic_retrieval/integration/test_semantic_retrieval.py index 5a32a9b2..4245f35c 100644 --- a/tests/semantic_retrieval/integration/test_semantic_retrieval.py +++ b/tests/semantic_retrieval/integration/test_semantic_retrieval.py @@ -1,7 +1,7 @@ """Real PostgreSQL vertical proof for maintenance, freshness and ranking.""" -from app.business.deployment_config import DeploymentConfigService -from app.business.info_base.services import RelationService +from inkcre_core.business.deployment_config import DeploymentConfigService +from inkcre_core.business.info_base.services import RelationService import os @@ -10,25 +10,25 @@ import pytest import sqlalchemy -from app.business.ai import AIManager -from app.business.info_base.resolver import register_core_resolvers -from app.business.semantic_retrieval import ( +from inkcre_core.business.ai import AIManager +from inkcre_core.business.info_base.resolver import register_core_resolvers +from inkcre_core.business.semantic_retrieval import ( EmbeddingProfileNotFoundError, SEMANTIC_RETRIEVAL_CONFIG_KEY, SEMANTIC_RETRIEVAL_CONFIG_SCHEMA, SemanticRetrievalManager, ) from tests.database import TestSession -from app.schemas.ai import ( +from inkcre_core.schemas.ai import ( AIModelModel, AIProviderModel, BlockEmbeddingModel, EmbeddingCapability, EmbeddingProfileModel, ) -from app.schemas.info_base.block import BlockModel -from app.schemas.info_base.relation import RelationModel -from app.schemas.semantic_retrieval import ( +from inkcre_core.schemas.info_base.block import BlockModel +from inkcre_core.schemas.info_base.relation import RelationModel +from inkcre_core.schemas.semantic_retrieval import ( EmbeddingMaintenanceOptions, VectorRetrievalOptions, ) diff --git a/tests/test_core_context.py b/tests/test_core_context.py new file mode 100644 index 00000000..3e221e22 --- /dev/null +++ b/tests/test_core_context.py @@ -0,0 +1,144 @@ +"""SDK bindings isolate concurrent deployments and preserve borrowed resources.""" + +import asyncio +from dataclasses import replace +from unittest.mock import AsyncMock, patch + +import pytest +from sqlalchemy.ext.asyncio import create_async_engine + +from inkcre_core import Core, CoreOptions, current_core +from inkcre_core.business.peer import PeerManager + + +def test_concurrent_instances_keep_their_identity_and_restore_nested_scope(async_runner): + first = Core("postgresql+psycopg://one:one@127.0.0.1:1/one") + second = Core("postgresql+psycopg://two:two@127.0.0.1:1/two") + caller = current_core() + + async def work(core, other): + with core.bind(): + assert PeerManager.get_current_peer_ref() == core.options.peer_id + with other.bind(): + assert current_core() is other + await asyncio.sleep(0) + assert current_core() is core + assert core.sources is not other.sources + assert core.active_jobs is not other.active_jobs + return PeerManager.get_current_peer_ref() + + async def run(): + try: + result = await asyncio.gather(work(first, second), work(second, first)) + assert result == [first.options.peer_id, second.options.peer_id] + finally: + await first.aclose() + await second.aclose() + + async_runner.run(run()) + assert current_core() is caller + with pytest.raises(RuntimeError, match="closed"): + with first.bind(): + pass + + +def test_borrowed_engine_is_not_disposed_and_options_are_explicit(async_runner): + engine = create_async_engine("postgresql+psycopg://borrowed:x@127.0.0.1:1/test") + options = CoreOptions(peer_name="external-consumer") + core = Core(engine, options=options) + with patch.object(type(engine), "dispose", new_callable=AsyncMock) as dispose: + async_runner.run(core.aclose()) + async_runner.run(core.aclose()) + dispose.assert_not_awaited() + async_runner.run(engine.dispose()) + assert core.options == options + assert replace(options, peer_name="another") != core.options + + +def test_owned_engine_is_disposed_once(async_runner): + core = Core("postgresql+psycopg://owned:x@127.0.0.1:1/test") + with patch.object(type(core.engine), "dispose", new_callable=AsyncMock) as dispose: + async_runner.run(core.aclose()) + async_runner.run(core.aclose()) + dispose.assert_awaited_once() + + +def test_legacy_imports_share_model_and_service_identity(): + from app.schemas.info_base.block import BlockModel as LegacyBlock + from inkcre_core.schemas.info_base.block import BlockModel + from app.business.info_base.main import InfoBaseManager as LegacyManager + from inkcre_core.business.info_base.main import InfoBaseManager + + assert LegacyBlock is BlockModel + assert LegacyManager is InfoBaseManager + + +def test_closing_one_core_drains_only_its_pending_work(async_runner): + first = Core("postgresql+psycopg://one:x@127.0.0.1:1/one") + second = Core("postgresql+psycopg://two:x@127.0.0.1:1/two") + + async def wait(): + await asyncio.Event().wait() + return True + + async def run(): + with first.bind(): + a = asyncio.create_task(wait()) + first.pending_jobs[1] = a + with second.bind(): + b = asyncio.create_task(wait()) + second.pending_jobs[2] = b + await first.aclose() + assert a.cancelled() + assert not b.done() + await second.aclose() + assert b.cancelled() + + async_runner.run(run()) + + +def test_asgi_binding_covers_lifespan_and_requests(async_runner): + from app.core import CoreMiddleware + + core = Core("postgresql+psycopg://test:x@127.0.0.1:1/test") + + async def application(scope, receive, send): + await asyncio.sleep(0) + assert current_core() is core + + async def receive(): + raise AssertionError("This probe does not receive HTTP data") + + async def send(message): + raise AssertionError("This probe does not send HTTP data") + + async def run(): + middleware = CoreMiddleware(application, core) + await middleware({"type": "lifespan"}, receive, send) + await middleware({"type": "http"}, receive, send) + await core.aclose() + + async_runner.run(run()) + + +def test_job_execution_keeps_logging_when_woken_by_a_quiet_poll(async_runner, monkeypatch): + from inkcre_core.business.job import JobManager + from inkcre_core.libs.obsrv.log_record import ENABLE_LOG_BACKEND, TRACE_ID + + async def execute(job_id): + assert job_id == 42 + assert ENABLE_LOG_BACKEND.get() + assert TRACE_ID.get() == "job.42" + return True + + monkeypatch.setattr(JobManager, "run", execute) + + async def run(): + token = ENABLE_LOG_BACKEND.set(False) + try: + assert await JobManager._run_traced(42) + assert not ENABLE_LOG_BACKEND.get() + finally: + ENABLE_LOG_BACKEND.reset(token) + + async_runner.run(run()) diff --git a/tests/test_extension_peer_status.py b/tests/test_extension_peer_status.py index 870d4648..f1836667 100644 --- a/tests/test_extension_peer_status.py +++ b/tests/test_extension_peer_status.py @@ -5,7 +5,7 @@ from app.business.extension.main import ExtensionHost from app.business.extension.state import ExtensionStore, InstalledExtension -from app.schemas.extension.main import ExtensionPeerStatus, SelectExtensionCommand +from inkcre_core.schemas.extension.main import ExtensionPeerStatus, SelectExtensionCommand def test_shared_protocol_preserves_a_distribution_unknown_to_python(): diff --git a/tests/test_extension_registry_config.py b/tests/test_extension_registry_config.py index 4b033439..e2b5ad82 100644 --- a/tests/test_extension_registry_config.py +++ b/tests/test_extension_registry_config.py @@ -2,14 +2,14 @@ import pytest -from app.business.deployment_config import DeploymentConfigService +from inkcre_core.business.deployment_config import DeploymentConfigService from app.business.extension.config import ( ExtensionRegistryDeploymentConfig, normalize_registry_origin, resolve_extension_registry_origin, ) -from app.business.peer import PeerManager -from app.schemas.peer import CorePeerConfig +from inkcre_core.business.peer import PeerManager +from inkcre_core.schemas.peer import CorePeerConfig from app.settings import settings diff --git a/tests/test_mcp_recall.py b/tests/test_mcp_recall.py index 4103eefc..c2018301 100644 --- a/tests/test_mcp_recall.py +++ b/tests/test_mcp_recall.py @@ -7,19 +7,22 @@ from mcp.client import Client from mcp.server import MCPServer -from app.business.info_base.resolver import register_core_resolvers -from app.business.lexical_retrieval import LexicalRetrievalManager -from app.business.semantic_retrieval import SemanticRetrievalManager +from inkcre_core.business.info_base.resolver import register_core_resolvers +from inkcre_core.business.lexical_retrieval import LexicalRetrievalManager +from inkcre_core.business.semantic_retrieval import SemanticRetrievalManager from app.business.sink.mcp import MCPSink -from app.schemas.info_base.block import BlockModel -from app.schemas.info_base.relation import RelationModel -from app.schemas.lexical_retrieval import LexicalRetrievalMatch, LexicalRetrievalResult -from app.schemas.semantic_retrieval import ( +from inkcre_core.schemas.info_base.block import BlockModel +from inkcre_core.schemas.info_base.relation import RelationModel +from inkcre_core.schemas.lexical_retrieval import ( + LexicalRetrievalMatch, + LexicalRetrievalResult, +) +from inkcre_core.schemas.semantic_retrieval import ( BlockSemanticRetrievalMatch, RelationSemanticRetrievalMatch, SemanticRetrievalResult, ) -from app.schemas.sink import SinkModel +from inkcre_core.schemas.sink import SinkModel def test_recall_labels_preserve_evidence_and_unreadable_candidates(monkeypatch): diff --git a/utils/__init__.py b/utils/__init__.py index e69de29b..bf76c59d 100644 --- a/utils/__init__.py +++ b/utils/__init__.py @@ -0,0 +1,3 @@ +"""Legacy import; use inkcre_core for new consumers.""" + +from inkcre_core.utils import * diff --git a/utils/base.py b/utils/base.py index f0fd8aae..bfa51a4b 100644 --- a/utils/base.py +++ b/utils/base.py @@ -1,15 +1,10 @@ -import enum -import aiohttp -import ssl -import certifi -import pydantic +"""Legacy import; use inkcre_core for new consumers.""" +from typing import TYPE_CHECKING +from importlib import import_module +import sys -enum_serializer = pydantic.PlainSerializer( - lambda value: value.value if isinstance(value, enum.Enum) else value, - return_type=str, -) +sys.modules[__name__] = import_module("inkcre_core.utils.base") - -def AIOHTTP_CONNECTOR_GETTER(): - return aiohttp.TCPConnector(ssl=ssl.create_default_context(cafile=certifi.where())) +if TYPE_CHECKING: + from inkcre_core.utils.base import * diff --git a/utils/datetime_.py b/utils/datetime_.py index a063f0ce..6113c7b5 100644 --- a/utils/datetime_.py +++ b/utils/datetime_.py @@ -1,96 +1,10 @@ -""" -author: Lan_zhijiang -date: 2024-11-10 -desc: 时间日期工具函数 -issues: +"""Legacy import; use inkcre_core for new consumers.""" -references: - https://git.hadream.ltd/anana/backend/common/-/wikis/Util/Datetime -""" +from typing import TYPE_CHECKING +from importlib import import_module +import sys -import datetime -import time +sys.modules[__name__] = import_module("inkcre_core.utils.datetime_") -timezone_delta: int = 8 # 默认东八区 - - -def get_timezone(timezone_delta: int = timezone_delta) -> datetime.timezone: - """ - 获取时区对象 - """ - return datetime.timezone(datetime.timedelta(hours=timezone_delta)) - - -def get_datetimez( - rfc3339: str | None = None, - timestamp: float | None = None, - iso8601: str | None = None, - timezone: datetime.timezone = get_timezone(), -) -> datetime.datetime: - """ - 获取当前时间对象（带时区） - - :param rfc3339: RFC3339格式时间字符串，如"2024-11-10T12:00:00+08:00" - :param timestamp: 时间戳（秒级浮点）；如果使用此，时区固定为 UTC+0 - :param iso8601: ISO8601格式时间字符串，如"2024-11-10 12:00:00" - :param timezone: 时区对象，可以通过 `get_timezone(8)` 获取东八区时区 - :raise ValueError: 如果rfc3339或iso8601都未提供，则抛出异常 - """ - if timestamp: - return datetime.datetime.fromtimestamp(timestamp, tz=get_timezone(0)) - if rfc3339 or iso8601: - val = rfc3339 or iso8601 - if val is None: - raise ValueError("rfc3339 or iso8601 must be provided") - return datetime.datetime.fromisoformat(val) - return datetime.datetime.now(tz=timezone) - - -def get_datetime(timestamp: float | None = None) -> datetime.datetime: - """ - 获取当前时间对象（不带时区） - """ - if timestamp: - return datetime.datetime.fromtimestamp(timestamp) - return datetime.datetime.now() - - -def get_timestamp( - datetime_obj: datetime.datetime | None = None, rfc3339: str | None = None -) -> float: - """ - 获取时间戳（秒级浮点） - """ - if datetime_obj: - return datetime_obj.timestamp() - if rfc3339: - return datetime.datetime.fromisoformat(rfc3339).timestamp() - return time.time() - - -def get_rfc3339( - timestamp: float | None = None, datetime_obj: datetime.datetime | None = None -) -> str: - """ - 获取RFC3339格式时间 - """ - if timestamp: - return datetime.datetime.fromtimestamp(timestamp).isoformat() - elif datetime_obj: - return datetime_obj.isoformat() - - raise ValueError("timestamp or datetime_obj must be provided") - - -def format_datetime(datetime: datetime.datetime, format: str = "%Y-%m-%d %H:%M:%S") -> str: - """ - 格式化时间对象 - """ - return datetime.strftime(format) - - -def format_timestamp(timestamp: float, format: str = "%Y-%m-%d %H:%M:%S") -> str: - """ - 格式化时间戳 - """ - return datetime.datetime.fromtimestamp(timestamp).strftime(format) +if TYPE_CHECKING: + from inkcre_core.utils.datetime_ import * diff --git a/utils/sql.py b/utils/sql.py index 6670ac97..239dd895 100644 --- a/utils/sql.py +++ b/utils/sql.py @@ -1,32 +1,10 @@ -"""Sql utilities.""" +"""Legacy import; use inkcre_core for new consumers.""" -import typing -import sqlalchemy -from sqlalchemy.dialects.postgresql import JSONB +from typing import TYPE_CHECKING +from importlib import import_module +import sys +sys.modules[__name__] = import_module("inkcre_core.utils.sql") -def find_by_json_field(column: typing.Any, field_name: str, value: str): - """Find by JSON field value. - - :param column: sqlalchemy column - - Examples: - >>> find_by_json_field(YourTable.column_name, "email", "example@example.com") - """ - return sqlalchemy.cast(column, JSONB)[field_name].astext == value - - -def find_by_nested_json(column: typing.Any, parent: str, child: str, value: str): - """Find by nested JSON field value. - - :param column: sqlalchemy column - """ - return sqlalchemy.cast(column, JSONB)[(parent, child)].astext == value - - -def find_by_json_contains(column: typing.Any, json_obj: dict): - """Find by JSON contains. - - :param column: sqlalchemy column - """ - return sqlalchemy.cast(column, JSONB).contains(json_obj) +if TYPE_CHECKING: + from inkcre_core.utils.sql import * diff --git a/utils/types_.py b/utils/types_.py index 60f22cd0..913bfc99 100644 --- a/utils/types_.py +++ b/utils/types_.py @@ -1,10 +1,10 @@ -__all__ = ["Undefined", "_undefined"] +"""Legacy import; use inkcre_core for new consumers.""" -import enum +from typing import TYPE_CHECKING +from importlib import import_module +import sys +sys.modules[__name__] = import_module("inkcre_core.utils.types_") -class Undefined(enum.Enum): - _undefined = 0 - - -_undefined = Undefined._undefined +if TYPE_CHECKING: + from inkcre_core.utils.types_ import *