fix: Update choreographer and rely on its message chunking

.github/workflows/publish_testpypi.yml

@@ -12,7 +12,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        python_v: ['3.8', '3.9', '3.10', '3.12', '3.13', '3.14']
+        python_v: ['3.9', '3.10', '3.12', '3.13', '3.14']
         # chrome_v: ['-1']
     defaults:
       run:

CHANGELOG.md

@@ -1,5 +1,15 @@
 ## Unreleased
 
+### Changed
+- Update Choreographer to v1.4.0 and use its large figure chunking, remove the Kaleido chunking code [[#478](https://github.com/plotly/Kaleido/pull/478)]
+
+### Removed
+- Drop support for Python 3.8 [[#478](https://github.com/plotly/Kaleido/pull/478)]
+- Remove the `orjson` dependency [[#478](https://github.com/plotly/Kaleido/pull/478)]
+
+### Fixed
+- Restore serialization of figure specs through the `plotly` JSON encoder [[#478](https://github.com/plotly/Kaleido/pull/478)]
+
 ## v1.4.0
 
 ### Fixed

src/py/.python-version

@@ -1 +1 @@
-3.8
+3.9

src/py/docs/examples_script.py

@@ -1,5 +1,5 @@
 # /// script
-# requires-python = ">=3.8"
+# requires-python = ">=3.9"
 # dependencies = [
 #    "pandas",
 #    "plotly[express]",

src/py/kaleido/_kaleido_tab/_tab.py

@@ -1,11 +1,9 @@
 from __future__ import annotations
 
 import base64
-from decimal import Decimal
 from typing import TYPE_CHECKING
 
 import logistro
-import orjson
 
 from . import _devtools_utils as _dtools
 from . import _js_logger
@@ -21,22 +19,10 @@
 
 
 _TEXT_FORMATS = ("svg", "json")  # eps
-_CHUNK_SIZE = 10 * 1024 * 1024  # 10 MB
 
 _logger = logistro.getLogger(__name__)
 
 
-def _orjson_default(obj):
-    """Fallback for types orjson can't handle natively (e.g. NumPy string arrays)."""
-    if isinstance(obj, Decimal):
-        return float(obj)
-    if hasattr(obj, "isoformat"):  # datetime-like, e.g. pandas Timestamp (#458)
-        return obj.isoformat()
-    if hasattr(obj, "tolist"):
-        return obj.tolist()
-    raise TypeError(f"Type is not JSON serializable: {type(obj).__name__}")
-
-
 def _subscribe_new(tab: choreo.Tab, event: str) -> asyncio.Future:
     """Create subscription to tab clearing old ones first: helper function."""
     new_future = tab.subscribe_once(event)
@@ -152,38 +138,23 @@ async def _calc_fig(
         render_prof,
         stepper,
     ) -> bytes:
-        render_prof.profile_log.tick("serializing spec")
-        spec_str = orjson.dumps(
-            spec,
-            default=_orjson_default,
-            option=orjson.OPT_SERIALIZE_NUMPY,
-        ).decode()
-        render_prof.profile_log.tick("spec serialized")
-
+        # Choreographer serializes the spec with the registered PlotlyJSONEncoder
+        # and splits messages that exceed the Chrome devtools buffer
+        kaleido_js_fn = (
+            r"function(spec, ...args)"
+            r"{"
+            r"return kaleido_scopes.plotly(spec, ...args).then(JSON.stringify);"
+            r"}"
+        )
         render_prof.profile_log.tick("sending javascript")
-        if len(spec_str) <= _CHUNK_SIZE:
-            kaleido_js_fn = (
-                r"function(specStr, ...args)"
-                r"{"
-                r"return kaleido_scopes"
-                r".plotly(JSON.parse(specStr), ...args)"
-                r".then(JSON.stringify);"
-                r"}"
-            )
-            result = await _dtools.exec_js_fn(
-                self.tab,
-                self._current_js_id,
-                kaleido_js_fn,
-                spec_str,
-                topojson,
-                stepper,
-            )
-        else:
-            result = await self._calc_fig_chunked(
-                spec_str,
-                topojson=topojson,
-                stepper=stepper,
-            )
+        result = await _dtools.exec_js_fn(
+            self.tab,
+            self._current_js_id,
+            kaleido_js_fn,
+            spec,
+            topojson,
+            stepper,
+        )
         _raise_error(result)
         render_prof.profile_log.tick("javascript sent")
 
@@ -205,45 +176,3 @@ async def _calc_fig(
         render_prof.data_out_size = len(res)
         render_prof.js_log = self.js_logger.log
         return res
-
-    async def _calc_fig_chunked(
-        self,
-        spec_str: str,
-        *,
-        topojson: str | None,
-        stepper,
-    ):
-        _raise_error(
-            await _dtools.exec_js_fn(
-                self.tab,
-                self._current_js_id,
-                r"function() { window.__kaleido_chunks = []; }",
-            )
-        )
-
-        for i in range(0, len(spec_str), _CHUNK_SIZE):
-            chunk = spec_str[i : i + _CHUNK_SIZE]
-            _raise_error(
-                await _dtools.exec_js_fn(
-                    self.tab,
-                    self._current_js_id,
-                    r"function(c) { window.__kaleido_chunks.push(c); }",
-                    chunk,
-                )
-            )
-
-        kaleido_js_fn = (
-            r"function(...args)"
-            r"{"
-            r"var spec = JSON.parse(window.__kaleido_chunks.join(''));"
-            r"delete window.__kaleido_chunks;"
-            r"return kaleido_scopes.plotly(spec, ...args).then(JSON.stringify);"
-            r"}"
-        )
-        return await _dtools.exec_js_fn(
-            self.tab,
-            self._current_js_id,
-            kaleido_js_fn,
-            topojson,
-            stepper,
-        )

src/py/kaleido/_page_generator.py

@@ -8,11 +8,11 @@
 from ._utils import path_tools
 
 if TYPE_CHECKING:
-    from typing import Tuple, Union
+    from typing import Union
 
     from typing_extensions import TypeAlias
 
-    UrlAndCharset: TypeAlias = Tuple[Union[str, Path], str]
+    UrlAndCharset: TypeAlias = tuple[Union[str, Path], str]
     """A tuple to explicitly set charset= in the <script> tag."""
 
 _logger = logistro.getLogger(__name__)

src/py/kaleido/_utils/__init__.py

@@ -12,7 +12,8 @@
 _logger = logistro.getLogger(__name__)
 
 if TYPE_CHECKING:
-    from typing import Any, AsyncIterator, Callable, Coroutine
+    from collections.abc import AsyncIterator, Coroutine
+    from typing import Any, Callable
 
 
 def event_printer(name: str) -> Callable[[Any], Coroutine[Any, Any, None]]:

src/py/kaleido/kaleido.py

@@ -20,16 +20,13 @@
 from ._utils import fig_tools, path_tools
 
 if TYPE_CHECKING:
+    from collections.abc import AsyncGenerator, ValuesView
     from types import TracebackType
     from typing import (
         Any,
-        AsyncGenerator,
-        List,
         Literal,
-        Tuple,
         TypeVar,
         Union,
-        ValuesView,
     )
 
     from typing_extensions import NotRequired, Required, TypeAlias, TypeGuard
@@ -39,7 +36,7 @@
 
     # union of sized iterables since 3.8 doesn't have & operator
     # Iterable & Sized
-    Listish: TypeAlias = Union[Tuple[T], List[T], ValuesView[T]]
+    Listish: TypeAlias = Union[tuple[T], list[T], ValuesView[T]]
 
     class FigureDict(TypedDict):
         """The type a fig_dicts returns for `write_fig_from_object`."""

src/py/kaleido/mocker/_utils.py

@@ -1,16 +1,16 @@
 from __future__ import annotations
 
 import itertools
+import json
 from pathlib import Path
 from typing import TYPE_CHECKING, TypedDict
 
 import logistro
-import orjson
 
 from ._args import args
 
 if TYPE_CHECKING:
-    from typing import Generator
+    from collections.abc import Generator
 
     from kaleido._utils.fig_tools import LayoutOpts
     from kaleido.kaleido import FigureDict
@@ -45,7 +45,7 @@ def load_figures_from_paths(paths: list[Path]) -> Generator[FigureDict, None]:
             raise RuntimeError(f"Path {path} is not a file.")
         _logger.info(f"Found file: {path!s}")
         with path.open(encoding="utf-8") as file:
-            figure = orjson.loads(file.read())
+            figure = json.loads(file.read())
             for f, w, h, s in itertools.product(  # all combos
                 args.format,
                 args.width,

src/py/pyproject.toml

@@ -16,7 +16,7 @@ name = "kaleido"
 description = "Plotly graph export library"
 license = { "file" = "LICENSE.md" }
 readme = "README.md"
-requires-python = ">=3.8"
+requires-python = ">=3.9"
 dynamic = ["version"]
 authors = [
   {name = "Andrew Pikul", email="ajpikul@gmail.com"},
@@ -26,9 +26,8 @@ maintainers = [
   {name = "Andrew Pikul", email = "ajpikul@gmail.com"},
 ]
 dependencies = [
-  "choreographer>=1.3.0",
+  "choreographer>=1.4.0",
   "logistro>=1.0.8",
-  "orjson>=3.10.15",
   "packaging",
 ]
 

src/py/tests/test_encoder_one_off.py

@@ -1,4 +1,8 @@
+import json
+from decimal import Decimal
+
 import numpy as np
+import pandas as pd
 import plotly.graph_objects as go
 import pytest
 from plotly.subplots import make_subplots
@@ -113,3 +117,33 @@ async def test_complex_plotly_encoder():
     img_bytes = await kaleido.calc_fig(fig)
 
     assert isinstance(img_bytes, bytes)
+
+
+@pytest.mark.parametrize(
+    ("trace_kwargs", "key", "expected"),
+    [
+        # Decimal values
+        ({"y": [Decimal("10.5"), Decimal(20)]}, "y", [10.5, 20]),
+        # pandas Timestamp values
+        (
+            {"x": [pd.Timestamp("2024-01-01"), pd.Timestamp("2024-01-02")]},
+            "x",
+            ["2024-01-01T00:00:00", "2024-01-02T00:00:00"],
+        ),
+        # pandas missing-value sentinels
+        (
+            {"x": [pd.Timestamp("2024-01-01"), pd.NaT]},
+            "x",
+            ["2024-01-01T00:00:00", None],
+        ),
+        ({"customdata": [1, pd.NA]}, "customdata", [1, None]),
+    ],
+    ids=["decimal", "timestamp", "nat", "na"],
+)
+async def test_plotly_encoder_types(trace_kwargs, key, expected):
+    """Test that kaleido serializes values the way the Plotly encoder does."""
+    fig = go.Figure(go.Scatter(**{"x": [1, 2], "y": [1, 2], **trace_kwargs}))
+
+    result = json.loads(await kaleido.calc_fig(fig, opts={"format": "json"}))
+
+    assert result["data"][0][key] == expected

src/py/tests/test_kaleido.py

@@ -12,7 +12,7 @@
 from kaleido import Kaleido
 
 if TYPE_CHECKING:
-    from typing import AsyncGenerator, Generator
+    from collections.abc import AsyncGenerator, Generator
 
     from kaleido import FigureDict