From fb700f76bcaffd0df0d2fc741d4bbf3e32cc287f Mon Sep 17 00:00:00 2001 From: tyeth Date: Wed, 9 Sep 2026 15:27:45 +0100 Subject: [PATCH] zephyr-cp: freeze .mpy modules named in circuitpython.toml The port had no frozen-module support: its Python-driven build never runs make, and the freeze pipeline lives in py/circuitpy_mpconfig.mk. Reimplement the pipeline in cptools/build_circuitpython.py behind a per-board opt-in, FROZEN_MPY_DIRS = ["frozen/", ...] in circuitpython.toml (paths relative to the repository root, as $(TOP)/... is in mpconfigboard.mk): - tools/preprocess_frozen_modules.py stages the trees into /frozen_mpy (repo directory dropped, __version__ filled in, examples and tests left out), before the qstr pass; - MICROPY_QSTR_EXTRA_POOL / MICROPY_MODULE_FROZEN_MPY are added to the flags for the qstr pass as well as the compile -- Q(.frozen) and the sys.path entry that uses it are behind MICROPY_MODULE_FROZEN; - after genhdr/qstrdefs.generated.h and root_pointers.h exist, mpy-cross compiles each module (-s with the module path, so mpy-tool derives the frozen name from it) and tools/mpy-tool.py -f -q emits frozen_content.c, fed the same collected qstr list that produced the generated header so the frozen pool numbers from MP_QSTRnumber_of correctly. tools/makemanifest.py is bypassed: it insists on genhdr/qstrdefs.preprocessed.h, which this builder never produces. Only MPY freezing: MICROPY_MODULE_FROZEN_STR would reference the mp_frozen_str_* tables makemanifest emits. - frozen_content.c is compiled with the no-qstr sources (it defines its own MP_QSTR_* enum values and must never go through extraction). pre_zephyr_build_prep.py builds mpy-cross first when a board freezes modules (honouring MICROPY_MPYCROSS), and tools/ci_fetch_deps.py learns the toml key so CI initialises the right frozen/ submodules (it had a TODO for this). The Pico W opts in with adafruit_ble: it has ~42 KB of heap and a BLE node cannot otherwise load the library. Measured on a Pico 2 W (same core, same flags): importing adafruit_ble plus its advertising.standard and services.nordic modules costs 10,384 B of heap frozen vs 21,792 B from .mpy files on CIRCUITPY (gc.mem_alloc() delta after gc.collect(); gc.mem_free() is not usable here, the split heap grows on demand), 0.055 s vs 0.132 s. Freezing the 20 modules adds 28,336 B of flash and no static RAM on the Pico W (1,208,636 -> 1,236,972 B with the rest of this series). The Pico 2 W is not opted in: it has the heap, and a frozen copy would pin the library version for everyone. Co-Authored-By: Claude Opus 5 --- .../rpi_pico_w_zephyr/circuitpython.toml | 5 + .../zephyr-cp/cptools/build_circuitpython.py | 98 +++++++++++++++++++ .../cptools/pre_zephyr_build_prep.py | 9 ++ tools/ci_fetch_deps.py | 10 +- 4 files changed, 120 insertions(+), 2 deletions(-) diff --git a/ports/zephyr-cp/boards/raspberrypi/rpi_pico_w_zephyr/circuitpython.toml b/ports/zephyr-cp/boards/raspberrypi/rpi_pico_w_zephyr/circuitpython.toml index deb99e3effa..5f22387d920 100644 --- a/ports/zephyr-cp/boards/raspberrypi/rpi_pico_w_zephyr/circuitpython.toml +++ b/ports/zephyr-cp/boards/raspberrypi/rpi_pico_w_zephyr/circuitpython.toml @@ -3,3 +3,8 @@ BLOBS=["hal_infineon"] # Non-Zephyr build of the same board; nvm and CIRCUITPY must sit where it puts them. counterpart = "raspberrypi/raspberry_pi_pico_w" + +# Frozen into flash: the Pico W has ~42 KB of heap, and adafruit_ble alone takes +# ~30 KB of it when loaded from CIRCUITPY as .mpy. Freezing keeps the bytecode in +# flash so a BLE node fits. +FROZEN_MPY_DIRS = ["frozen/Adafruit_CircuitPython_BLE"] diff --git a/ports/zephyr-cp/cptools/build_circuitpython.py b/ports/zephyr-cp/cptools/build_circuitpython.py index 8594c245c81..6172e45f35c 100644 --- a/ports/zephyr-cp/cptools/build_circuitpython.py +++ b/ports/zephyr-cp/cptools/build_circuitpython.py @@ -1,7 +1,10 @@ import asyncio import logging import os +import os import pathlib +import shutil +import subprocess import pickle import sys @@ -356,6 +359,77 @@ def determine_enabled_modules(board_info, portdir, srcdir): return enabled_modules, module_reasons +def stage_frozen_modules(frozen_dirs, srcdir, builddir): + """Copy the boards' frozen library trees into builddir/frozen_mpy. + + Mirrors the ``$(BUILD)/frozen_mpy`` step of py/circuitpy_mpconfig.mk: the + repo-name directory is dropped, ``__version__`` is filled in and examples, + docs and tests are left out. Returns the staged .py files, relative to the + staging directory, in a stable order. + """ + staging = builddir / "frozen_mpy" + if staging.exists(): + shutil.rmtree(staging) + staging.mkdir(parents=True) + env = dict(os.environ) + env["PYTHONPATH"] = str(srcdir / "tools" / "python-semver") + subprocess.run( + [ + sys.executable, + srcdir / "tools" / "preprocess_frozen_modules.py", + "-o", + staging, + *[srcdir / d for d in frozen_dirs], + ], + cwd=srcdir, + env=env, + check=True, + ) + return sorted(p.relative_to(staging) for p in staging.rglob("*.py")) + + +def freeze_modules(frozen_sources, srcdir, builddir, mpy_cross, qstr_defs): + """Compile the staged modules with mpy-cross and emit frozen_content.c. + + The make flow drives this through tools/makemanifest.py; that script insists + on ``$(BUILD)/genhdr/qstrdefs.preprocessed.h``, which this builder never + produces (it feeds the collected qstrs straight to makeqstrdata.py), so the + two tools it wraps are invoked directly. ``qstr_defs`` must be the exact + file that produced genhdr/qstrdefs.generated.h: mpy-tool numbers the frozen + modules' extra qstrs from MP_QSTRnumber_of onwards, so the two pools must be + computed from the same set. + """ + staging = builddir / "frozen_mpy" + mpy_files = [] + for rel in frozen_sources: + out = staging / rel.with_suffix(".mpy") + # -s records the module path (not the staging path) as the source name, + # which is what mpy-tool derives the frozen module name from. + subprocess.run( + [mpy_cross, "-s", str(rel), "-o", out, staging / rel], + cwd=staging, + check=True, + ) + mpy_files.append(out) + frozen_content = builddir / "frozen_content.c" + with frozen_content.open("w") as f: + subprocess.run( + [ + sys.executable, + srcdir / "tools" / "mpy-tool.py", + "-f", + "-q", + qstr_defs, + "-mlongint-impl=mpz", + *mpy_files, + ], + cwd=srcdir, + stdout=f, + check=True, + ) + return frozen_content + + async def build_circuitpython(): # noqa: C901 circuitpython_flags = ["-DCIRCUITPY"] port_flags = [] @@ -404,6 +478,22 @@ async def build_circuitpython(): # noqa: C901 if mpconfigboard_fn is not None and mpconfigboard_fn.exists(): with mpconfigboard_fn.open("rb") as f: mpconfigboard.update(tomllib.load(f)) + # Frozen modules (opt-in per board: FROZEN_MPY_DIRS in circuitpython.toml, + # paths relative to the repository root, as $(TOP)/... is in mpconfigboard.mk). + # The flags have to be present for the qstr pass as well as the compile: + # MICROPY_MODULE_FROZEN gates both Q(.frozen) and the sys.path entry that + # uses it, and the extra pool is how frozen qstrs get their numbers. + frozen_dirs = mpconfigboard.get("FROZEN_MPY_DIRS", []) + frozen_sources = [] + if frozen_dirs: + circuitpython_flags.append("-DMICROPY_QSTR_EXTRA_POOL=mp_qstr_frozen_const_pool") + # Only .mpy freezing: MICROPY_MODULE_FROZEN_STR would make frozenmod.c + # reference the mp_frozen_str_* tables that tools/makemanifest.py + # emits, and mpy-tool alone does not. + circuitpython_flags.append("-DMICROPY_MODULE_FROZEN_MPY=1") + frozen_sources = stage_frozen_modules(frozen_dirs, srcdir, builddir) + logger.info(f"Freezing {len(frozen_sources)} modules from {', '.join(frozen_dirs)}") + async with asyncio.TaskGroup() as tg: tg.create_task( cpbuild.run_command( @@ -736,6 +826,14 @@ async def build_circuitpython(): # noqa: C901 # This file is generated by the QSTR/translation process. source_files.append(builddir / f"translations-{translation}.c") + if frozen_dirs: + # Needs genhdr/qstrdefs.generated.h and root_pointers.h from the task + # group above. frozen_content.c defines its own MP_QSTR_* enum values, so + # it must never go through the qstr extraction pass. + mpy_cross = os.environ.get("MICROPY_MPYCROSS", str(srcdir / "mpy-cross" / "build" / "mpy-cross")) + source_files.append( + freeze_modules(frozen_sources, srcdir, builddir, mpy_cross, builddir / "qstrdefs.collected") + ) # These files don't include unique QSTRs. They just need to be compiled. source_files.append(portdir / "supervisor" / "flash.c") source_files.append(portdir / "supervisor" / "port.c") diff --git a/ports/zephyr-cp/cptools/pre_zephyr_build_prep.py b/ports/zephyr-cp/cptools/pre_zephyr_build_prep.py index f42fc1a3a85..4795aaabc66 100644 --- a/ports/zephyr-cp/cptools/pre_zephyr_build_prep.py +++ b/ports/zephyr-cp/cptools/pre_zephyr_build_prep.py @@ -1,4 +1,5 @@ # Called by the Makefile before calling out to `west`. +import os import pathlib import subprocess import sys @@ -20,6 +21,14 @@ args = blob_fetch_args.get(blob, []) subprocess.run(["west", "blobs", "fetch", blob, *args], check=True) +# Frozen modules need the host mpy-cross; build it up front, where make is +# already in use, rather than from inside the CMake-driven CircuitPython step. +if mpconfigboard.get("FROZEN_MPY_DIRS") and "MICROPY_MPYCROSS" not in os.environ: + subprocess.run( + ["make", "-C", str(portdir.parent.parent / "mpy-cross"), "USER_C_MODULES="], + check=True, + ) + if board.endswith("bsim"): subprocess.run( ["make", "everything", "-j", "8"], diff --git a/tools/ci_fetch_deps.py b/tools/ci_fetch_deps.py index 8994e54c338..cfb35af5ca2 100644 --- a/tools/ci_fetch_deps.py +++ b/tools/ci_fetch_deps.py @@ -5,6 +5,7 @@ import pathlib import re import subprocess +import tomllib TOP = pathlib.Path(__file__).parent.parent @@ -254,8 +255,13 @@ def main(target): lib_folder = "/".join(lib_folder[:2]) submodules.append(lib_folder) else: - # TODO: Add a way to specify frozen modules in circuitpython.toml - pass + # ports/zephyr-cp: FROZEN_MPY_DIRS = ["frozen/", ...] in circuitpython.toml + with config.open("rb") as f: + board_config = tomllib.load(f) + for lib_folder in board_config.get("FROZEN_MPY_DIRS", []): + if lib_folder.count("/") > 1: + lib_folder = "/".join(lib_folder.split("/", maxsplit=2)[:2]) + submodules.append(lib_folder) print("Submodules:", " ".join(submodules))