perf: reduce image memory overhead across lifecycle

astrbot/core/agent/context/compressor.py

@@ -1,10 +1,20 @@
+from pathlib import Path
 from typing import TYPE_CHECKING, Protocol, runtime_checkable
 
+from astrbot.core.exceptions import ProviderRequestTooLargeError
+from astrbot.core.utils.astrbot_path import get_astrbot_data_path
+from astrbot.core.utils.image_media_store import (
+    ImageMediaStore,
+    materialize_image_media_refs,
+)
+from astrbot.core.utils.media_utils import ImagePayloadTooLargeError
+
 from ...provider.modalities import (
     log_context_sanitize_stats,
     sanitize_contexts_by_modalities,
 )
 from ..message import Message
+from .image_budget import get_image_encoded_byte_limit, validate_context_image_bytes
 from .token_counter import EstimateTokenCounter, TokenCounter
 
 if TYPE_CHECKING:
@@ -130,6 +140,7 @@ def __init__(
         instruction_text: str | None = None,
         compression_threshold: float = 0.82,
         token_counter: TokenCounter | None = None,
+        image_media_store: ImageMediaStore | None = None,
     ) -> None:
         """Initialize the LLM summary compressor.
 
@@ -139,8 +150,10 @@ def __init__(
                 exact context. Clamped to 0-0.3.
             instruction_text: Custom instruction for summary generation.
             compression_threshold: The compression trigger threshold (default: 0.82).
+            image_media_store: Store shared with the main request.
         """
         self.provider = provider
+        self.image_media_store = image_media_store
         self.keep_recent_ratio = min(max(float(keep_recent_ratio), 0.0), 0.3)
         self.compression_threshold = compression_threshold
         self.token_counter = token_counter or EstimateTokenCounter()
@@ -273,11 +286,29 @@ async def __call__(self, messages: list[Message]) -> list[Message]:
 
         # Generate summary
         try:
+            limit = get_image_encoded_byte_limit(
+                getattr(self.provider, "provider_settings", {})
+            )
+            validate_context_image_bytes(sanitized_summary_contexts, limit)
+            sanitized_summary_contexts = await materialize_image_media_refs(
+                sanitized_summary_contexts,
+                self.image_media_store
+                or ImageMediaStore(Path(get_astrbot_data_path()) / "media"),
+            )
             response = await self.provider.text_chat(
                 contexts=sanitized_summary_contexts,
             )
             summary_content = (response.completion_text or "").strip()
+        except (MemoryError, ImagePayloadTooLargeError, ProviderRequestTooLargeError):
+            raise
         except Exception as e:
+            status_code = getattr(e, "status_code", None) or getattr(
+                getattr(e, "response", None), "status_code", None
+            )
+            if status_code == 413:
+                raise ProviderRequestTooLargeError(
+                    "The summary provider rejected the request size (HTTP 413)."
+                ) from e
             logger.error(f"Failed to generate summary: {e}")
             return messages
 

astrbot/core/agent/context/config.py

@@ -6,6 +6,7 @@
 
 if TYPE_CHECKING:
     from astrbot.core.provider.provider import Provider
+    from astrbot.core.utils.image_media_store import ImageMediaStore
 
 
 @dataclass
@@ -33,3 +34,5 @@ class ContextConfig:
     """Custom token counting method. If None, the default method is used."""
     custom_compressor: ContextCompressor | None = None
     """Custom context compression method. If None, the default method is used."""
+    image_media_store: "ImageMediaStore | None" = None
+    """Application media store shared by main and summary requests."""

astrbot/core/agent/context/image_budget.py

@@ -0,0 +1,87 @@
+"""Validate image bytes independently from text-token estimates."""
+
+from collections.abc import Sequence
+
+from astrbot.core.agent.message import ImageMediaRefPart, ImageURLPart, Message
+from astrbot.core.utils.media_utils import (
+    IMAGE_COMPRESS_DEFAULT_MAX_ENCODED_BYTES,
+    ImagePayloadTooLargeError,
+)
+
+
+def get_image_encoded_byte_limit(provider_settings: dict | None) -> int:
+    """Read the configured per-image limit with the same rules for every request.
+
+    Args:
+        provider_settings: Settings belonging to the actual request provider.
+
+    Returns:
+        A positive byte limit, or the default when the setting is invalid.
+    """
+    options = (
+        provider_settings.get("image_compress_options", {})
+        if isinstance(provider_settings, dict)
+        else {}
+    )
+    limit = options.get("max_encoded_bytes") if isinstance(options, dict) else None
+    if isinstance(limit, int) and not isinstance(limit, bool) and limit > 0:
+        return limit
+    return IMAGE_COMPRESS_DEFAULT_MAX_ENCODED_BYTES
+
+
+def validate_context_image_bytes(
+    messages: Sequence[Message | dict],
+    max_encoded_bytes: int = IMAGE_COMPRESS_DEFAULT_MAX_ENCODED_BYTES,
+) -> int:
+    """Validate selected images without loading or rewriting historical bytes.
+
+    Args:
+        messages: Selected main or summary request messages.
+        max_encoded_bytes: Maximum Base64 bytes for a single image.
+
+    Returns:
+        Total known encoded-image bytes, excluding JSON and data URI headers.
+        Remote URLs have unknown size until resolved and are not counted.
+
+    Raises:
+        ImagePayloadTooLargeError: A selected image exceeds the single-image cap.
+        ValueError: The configured cap is invalid.
+    """
+    if isinstance(max_encoded_bytes, bool) or max_encoded_bytes < 1:
+        raise ValueError("Image byte budget must be a positive integer")
+    total = 0
+    for message in messages:
+        parts = (
+            message.content if isinstance(message, Message) else message.get("content")
+        )
+        if not isinstance(parts, list):
+            continue
+        for part in parts:
+            size = 0
+            url = None
+            if isinstance(part, ImageMediaRefPart):
+                size = 4 * ((part.byte_size + 2) // 3)
+            elif isinstance(part, ImageURLPart):
+                url = part.image_url.url
+            elif isinstance(part, dict):
+                if part.get("type") == "image_media_ref":
+                    size = 4 * ((int(part["byte_size"]) + 2) // 3)
+                elif part.get("type") == "image_url":
+                    image_url = part.get("image_url")
+                    url = (
+                        image_url.get("url")
+                        if isinstance(image_url, dict)
+                        else image_url
+                    )
+            if isinstance(url, str) and url.startswith("data:image/"):
+                comma = url.find(",")
+                if comma >= 0 and ";base64" in url[:comma]:
+                    # Do not slice the full Base64 suffix merely to count it.
+                    size = len(url) - comma - 1
+            if size > max_encoded_bytes:
+                raise ImagePayloadTooLargeError(
+                    f"A selected image uses {size} Base64 bytes, exceeding the "
+                    f"{max_encoded_bytes}-byte limit. Historical images were not changed."
+                )
+            total += size
+    return total

astrbot/core/agent/context/manager.py

@@ -1,4 +1,6 @@
 from astrbot import logger
+from astrbot.core.exceptions import ProviderRequestTooLargeError
+from astrbot.core.utils.media_utils import ImagePayloadTooLargeError
 
 from ..message import Message
 from .compressor import LLMSummaryCompressor, TruncateByTurnsCompressor
@@ -36,6 +38,7 @@ def __init__(
                 keep_recent_ratio=config.llm_compress_keep_recent_ratio,
                 instruction_text=config.llm_compress_instruction,
                 token_counter=self.token_counter,
+                image_media_store=config.image_media_store,
             )
         else:
             self.compressor = TruncateByTurnsCompressor(
@@ -76,6 +79,8 @@ async def process(
                     result = await self._run_compression(result, total_tokens)
 
             return result
+        except (MemoryError, ImagePayloadTooLargeError, ProviderRequestTooLargeError):
+            raise
         except Exception as e:
             logger.error(f"Error during context processing: {e}", exc_info=True)
             return messages

astrbot/core/agent/context/token_counter.py

@@ -1,7 +1,14 @@
 import json
 from typing import Protocol, runtime_checkable
 
-from ..message import AudioURLPart, ImageURLPart, Message, TextPart, ThinkPart
+from ..message import (
+    AudioURLPart,
+    ImageMediaRefPart,
+    ImageURLPart,
+    Message,
+    TextPart,
+    ThinkPart,
+)
 
 
 @runtime_checkable
@@ -60,7 +67,7 @@ def count_tokens(
                         total += self._estimate_tokens(part.text)
                     elif isinstance(part, ThinkPart):
                         total += self._estimate_tokens(part.think)
-                    elif isinstance(part, ImageURLPart):
+                    elif isinstance(part, (ImageURLPart, ImageMediaRefPart)):
                         total += IMAGE_TOKEN_ESTIMATE
                     elif isinstance(part, AudioURLPart):
                         total += AUDIO_TOKEN_ESTIMATE

astrbot/core/agent/message.py

@@ -120,11 +120,33 @@ class ImageURL(BaseModel):
         """The URL of the image, can be data URI scheme like `data:image/png;base64,...`."""
         id: str | None = None
         """The ID of the image, to allow LLMs to distinguish different images."""
+        detail: str | None = None
+
+        @model_serializer(mode="wrap")
+        def serialize(self, handler):
+            data = handler(self)
+            if self.detail is None:
+                data.pop("detail", None)
+            return data
 
     type: str = "image_url"
     image_url: ImageURL
 
 
+class ImageMediaRefPart(ContentPart):
+    """A durable image reference that is resolved only for a provider request."""
+
+    type: str = "image_media_ref"
+    media_id: str
+    mime_type: str
+    width: int | None = None
+    height: int | None = None
+    byte_size: int
+    detail: str | None = None
+    version: int = 1
+    image_id: str | None = None
+
+
 class AudioURLPart(ContentPart):
     """
     >>> AudioURLPart(audio_url=AudioURLPart.AudioURL(url="https://example.com/audio.mp3")).model_dump()