zephyr-cp: freeze .mpy modules named in circuitpython.toml

ports/zephyr-cp/boards/raspberrypi/rpi_pico_w_zephyr/circuitpython.toml

@@ -3,3 +3,8 @@ BLOBS=["hal_infineon"]
 
 # Non-Zephyr build of the same board; nvm and CIRCUITPY must sit where it puts them.
 counterpart = "raspberrypi/raspberry_pi_pico_w"
+
+# Frozen into flash: the Pico W has ~42 KB of heap, and adafruit_ble alone takes
+# ~30 KB of it when loaded from CIRCUITPY as .mpy. Freezing keeps the bytecode in
+# flash so a BLE node fits.
+FROZEN_MPY_DIRS = ["frozen/Adafruit_CircuitPython_BLE"]

ports/zephyr-cp/cptools/build_circuitpython.py

@@ -1,7 +1,10 @@
 import asyncio
 import logging
 import os
+import os
 import pathlib
+import shutil
+import subprocess
 import pickle
 import sys
 
@@ -356,6 +359,77 @@ def determine_enabled_modules(board_info, portdir, srcdir):
     return enabled_modules, module_reasons
 
 
+def stage_frozen_modules(frozen_dirs, srcdir, builddir):
+    """Copy the boards' frozen library trees into builddir/frozen_mpy.
+
+    Mirrors the ``$(BUILD)/frozen_mpy`` step of py/circuitpy_mpconfig.mk: the
+    repo-name directory is dropped, ``__version__`` is filled in and examples,
+    docs and tests are left out. Returns the staged .py files, relative to the
+    staging directory, in a stable order.
+    """
+    staging = builddir / "frozen_mpy"
+    if staging.exists():
+        shutil.rmtree(staging)
+    staging.mkdir(parents=True)
+    env = dict(os.environ)
+    env["PYTHONPATH"] = str(srcdir / "tools" / "python-semver")
+    subprocess.run(
+        [
+            sys.executable,
+            srcdir / "tools" / "preprocess_frozen_modules.py",
+            "-o",
+            staging,
+            *[srcdir / d for d in frozen_dirs],
+        ],
+        cwd=srcdir,
+        env=env,
+        check=True,
+    )
+    return sorted(p.relative_to(staging) for p in staging.rglob("*.py"))
+
+
+def freeze_modules(frozen_sources, srcdir, builddir, mpy_cross, qstr_defs):
+    """Compile the staged modules with mpy-cross and emit frozen_content.c.
+
+    The make flow drives this through tools/makemanifest.py; that script insists
+    on ``$(BUILD)/genhdr/qstrdefs.preprocessed.h``, which this builder never
+    produces (it feeds the collected qstrs straight to makeqstrdata.py), so the
+    two tools it wraps are invoked directly. ``qstr_defs`` must be the exact
+    file that produced genhdr/qstrdefs.generated.h: mpy-tool numbers the frozen
+    modules' extra qstrs from MP_QSTRnumber_of onwards, so the two pools must be
+    computed from the same set.
+    """
+    staging = builddir / "frozen_mpy"
+    mpy_files = []
+    for rel in frozen_sources:
+        out = staging / rel.with_suffix(".mpy")
+        # -s records the module path (not the staging path) as the source name,
+        # which is what mpy-tool derives the frozen module name from.
+        subprocess.run(
+            [mpy_cross, "-s", str(rel), "-o", out, staging / rel],
+            cwd=staging,
+            check=True,
+        )
+        mpy_files.append(out)
+    frozen_content = builddir / "frozen_content.c"
+    with frozen_content.open("w") as f:
+        subprocess.run(
+            [
+                sys.executable,
+                srcdir / "tools" / "mpy-tool.py",
+                "-f",
+                "-q",
+                qstr_defs,
+                "-mlongint-impl=mpz",
+                *mpy_files,
+            ],
+            cwd=srcdir,
+            stdout=f,
+            check=True,
+        )
+    return frozen_content
+
+
 async def build_circuitpython():  # noqa: C901
     circuitpython_flags = ["-DCIRCUITPY"]
     port_flags = []
@@ -404,6 +478,22 @@ async def build_circuitpython():  # noqa: C901
     if mpconfigboard_fn is not None and mpconfigboard_fn.exists():
         with mpconfigboard_fn.open("rb") as f:
             mpconfigboard.update(tomllib.load(f))
+    # Frozen modules (opt-in per board: FROZEN_MPY_DIRS in circuitpython.toml,
+    # paths relative to the repository root, as $(TOP)/... is in mpconfigboard.mk).
+    # The flags have to be present for the qstr pass as well as the compile:
+    # MICROPY_MODULE_FROZEN gates both Q(.frozen) and the sys.path entry that
+    # uses it, and the extra pool is how frozen qstrs get their numbers.
+    frozen_dirs = mpconfigboard.get("FROZEN_MPY_DIRS", [])
+    frozen_sources = []
+    if frozen_dirs:
+        circuitpython_flags.append("-DMICROPY_QSTR_EXTRA_POOL=mp_qstr_frozen_const_pool")
+        # Only .mpy freezing: MICROPY_MODULE_FROZEN_STR would make frozenmod.c
+        # reference the mp_frozen_str_* tables that tools/makemanifest.py
+        # emits, and mpy-tool alone does not.
+        circuitpython_flags.append("-DMICROPY_MODULE_FROZEN_MPY=1")
+        frozen_sources = stage_frozen_modules(frozen_dirs, srcdir, builddir)
+        logger.info(f"Freezing {len(frozen_sources)} modules from {', '.join(frozen_dirs)}")
+
     async with asyncio.TaskGroup() as tg:
         tg.create_task(
             cpbuild.run_command(
@@ -736,6 +826,14 @@ async def build_circuitpython():  # noqa: C901
 
     # This file is generated by the QSTR/translation process.
     source_files.append(builddir / f"translations-{translation}.c")
+    if frozen_dirs:
+        # Needs genhdr/qstrdefs.generated.h and root_pointers.h from the task
+        # group above. frozen_content.c defines its own MP_QSTR_* enum values, so
+        # it must never go through the qstr extraction pass.
+        mpy_cross = os.environ.get("MICROPY_MPYCROSS", str(srcdir / "mpy-cross" / "build" / "mpy-cross"))
+        source_files.append(
+            freeze_modules(frozen_sources, srcdir, builddir, mpy_cross, builddir / "qstrdefs.collected")
+        )
     # These files don't include unique QSTRs. They just need to be compiled.
     source_files.append(portdir / "supervisor" / "flash.c")
     source_files.append(portdir / "supervisor" / "port.c")

ports/zephyr-cp/cptools/pre_zephyr_build_prep.py

@@ -1,4 +1,5 @@
 # Called by the Makefile before calling out to `west`.
+import os
 import pathlib
 import subprocess
 import sys
@@ -20,6 +21,14 @@
     args = blob_fetch_args.get(blob, [])
     subprocess.run(["west", "blobs", "fetch", blob, *args], check=True)
 
+# Frozen modules need the host mpy-cross; build it up front, where make is
+# already in use, rather than from inside the CMake-driven CircuitPython step.
+if mpconfigboard.get("FROZEN_MPY_DIRS") and "MICROPY_MPYCROSS" not in os.environ:
+    subprocess.run(
+        ["make", "-C", str(portdir.parent.parent / "mpy-cross"), "USER_C_MODULES="],
+        check=True,
+    )
+
 if board.endswith("bsim"):
     subprocess.run(
         ["make", "everything", "-j", "8"],

tools/ci_fetch_deps.py

@@ -5,6 +5,7 @@
 import pathlib
 import re
 import subprocess
+import tomllib
 
 TOP = pathlib.Path(__file__).parent.parent
 
@@ -254,8 +255,13 @@ def main(target):
                             lib_folder = "/".join(lib_folder[:2])
                         submodules.append(lib_folder)
         else:
-            # TODO: Add a way to specify frozen modules in circuitpython.toml
-            pass
+            # ports/zephyr-cp: FROZEN_MPY_DIRS = ["frozen/<lib>", ...] in circuitpython.toml
+            with config.open("rb") as f:
+                board_config = tomllib.load(f)
+            for lib_folder in board_config.get("FROZEN_MPY_DIRS", []):
+                if lib_folder.count("/") > 1:
+                    lib_folder = "/".join(lib_folder.split("/", maxsplit=2)[:2])
+                submodules.append(lib_folder)
 
     print("Submodules:", " ".join(submodules))