From edc0205cf2a4ed59763c2a282546a3fea7b40171 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20S=C5=82uszniak?= Date: Thu, 27 Aug 2026 17:15:38 +0200 Subject: [PATCH 01/24] feat(models)!: resolve DEFAULT model variant per platform `DEFAULT` pointed at a fixed export, almost always the universal XNNPACK one, so an iOS app that did nothing special ran on the CPU while the Neural Engine sat idle. It now resolves when the library loads, from the platform, the backends the binary was actually linked with, and the order the variants are declared in. - iOS device: the Core ML export where one exists. - Android and the iOS simulator: XNNPACK. - Narrowed by the app's `react-native-executorch` block, so opting out of a backend moves `DEFAULT` back rather than failing to load. - MLX and Vulkan stay an explicit opt-in. Also fixes `instanceSegmentation.RFDETR_NANO`, which defaulted to a Core ML file on Android, and refreshes the `features` -> backend map, which omitted coreml for semanticSegmentation, imageEmbeddings and textToImage, and mlx for speechToText and textToSpeech. A backend a feature does not provision is one `DEFAULT` can never resolve to, so a test now holds the map in sync with the registry. Closes #1389 --- apps/computer-vision/app/detection/index.tsx | 4 +- apps/computer-vision/app/keypoint/index.tsx | 2 +- apps/computer-vision/package.json | 4 +- apps/speech/package.json | 1 + .../01-fundamentals/01-getting-started.md | 38 +- .../__tests__/api/modelVariants.test.ts | 325 +++++++++ .../scripts/download-libs.js | 39 +- .../src/modelVariants.ts | 168 +++++ .../react-native-executorch/src/models.ts | 620 ++++++++---------- 9 files changed, 816 insertions(+), 385 deletions(-) create mode 100644 packages/react-native-executorch/__tests__/api/modelVariants.test.ts create mode 100644 packages/react-native-executorch/src/modelVariants.ts diff --git a/apps/computer-vision/app/detection/index.tsx b/apps/computer-vision/app/detection/index.tsx index fdee01d0a1..ba5410a077 100644 --- a/apps/computer-vision/app/detection/index.tsx +++ b/apps/computer-vision/app/detection/index.tsx @@ -16,11 +16,11 @@ import { BoundingBox } from '../../components/BoundingBox'; const MODEL_OPTIONS: ModelOption[] = [ { label: 'SSDLite 320 MobileNet V3 Large (XNNPACK FP32)', - value: models.objectDetection.SSDLITE320_MOBILENET_V3_LARGE.DEFAULT, + value: models.objectDetection.SSDLITE320_MOBILENET_V3_LARGE.XNNPACK_FP32, }, { label: 'RF-DETR Nano (XNNPACK FP32)', - value: models.objectDetection.RFDETR_NANO.DEFAULT, + value: models.objectDetection.RFDETR_NANO.XNNPACK_FP32, }, { label: 'RF-DETR Nano (CoreML FP16)', diff --git a/apps/computer-vision/app/keypoint/index.tsx b/apps/computer-vision/app/keypoint/index.tsx index d0649996da..cc6fce3309 100644 --- a/apps/computer-vision/app/keypoint/index.tsx +++ b/apps/computer-vision/app/keypoint/index.tsx @@ -16,7 +16,7 @@ import { BoundingBox } from '../../components/BoundingBox'; const MODEL_OPTIONS: ModelOption[] = [ { label: 'BlazeFace (XNNPACK FP32)', - value: models.keypointDetection.BLAZEFACE.DEFAULT, + value: models.keypointDetection.BLAZEFACE.XNNPACK_FP32, }, { label: 'YOLO26 Pose (XNNPACK FP32)', diff --git a/apps/computer-vision/package.json b/apps/computer-vision/package.json index d26e149908..babc706ba8 100644 --- a/apps/computer-vision/package.json +++ b/apps/computer-vision/package.json @@ -10,8 +10,10 @@ "instanceSegmentation", "keypointDetection", "objectDetection", + "ocr", "semanticSegmentation", - "styleTransfer" + "styleTransfer", + "textToImage" ] }, "scripts": { diff --git a/apps/speech/package.json b/apps/speech/package.json index 23d387d2df..55961aa887 100644 --- a/apps/speech/package.json +++ b/apps/speech/package.json @@ -5,6 +5,7 @@ "react-native-executorch": { "features": [ "vad", + "speechToText", "textToSpeech" ] }, diff --git a/docs/docs/01-fundamentals/01-getting-started.md b/docs/docs/01-fundamentals/01-getting-started.md index 3c9e2c5603..61e449936b 100644 --- a/docs/docs/01-fundamentals/01-getting-started.md +++ b/docs/docs/01-fundamentals/01-getting-started.md @@ -213,17 +213,47 @@ The three lists are merged, so you can pair a `features` set with an extra `back ### Feature → backend / lib mapping -These tasks are available today: +Each feature provisions the union of the backends its models are published for, so that the `DEFAULT` variant of every model in that family can resolve to the fastest export the device supports (see below). | Feature | Backends | Extra libs | | --- | --- | --- | +| `llm` | xnnpack, mlx | — | +| `multimodalLLM` | xnnpack, mlx, vulkan | opencv | +| `privacyFilter` | xnnpack, mlx | — | +| `speechToText` | xnnpack, coreml, mlx | — | +| `textToSpeech` | xnnpack, mlx | phonemis | +| `vad` | xnnpack | — | +| `textEmbeddings` | xnnpack, mlx | — | +| `imageEmbeddings` | xnnpack, coreml, mlx | opencv | | `classification` | xnnpack, coreml | opencv | -| `semanticSegmentation` | xnnpack | opencv | -| `styleTransfer` | xnnpack, coreml | opencv | +| `objectDetection` | xnnpack, coreml | opencv | | `keypointDetection` | xnnpack, coreml, mlx | opencv | +| `semanticSegmentation` | xnnpack, coreml | opencv | +| `instanceSegmentation` | xnnpack, coreml | opencv | +| `ocr` | xnnpack, coreml, vulkan | opencv | +| `verticalOCR` | xnnpack | opencv | +| `styleTransfer` | xnnpack, coreml | opencv | +| `textToImage` | xnnpack, coreml | opencv | +| `segmentAnything` | xnnpack, coreml | opencv | | `tokenizer` | — | — | -The map also contains forward-looking entries (`llm`, `multimodalLLM`, `speechToText`, `objectDetection`, `ocr`, …) for tasks that are not yet exposed in the JS API; requesting one provisions the right binaries but has no hook to call yet. +### How the backends you pick change which model runs + +Models published for more than one backend expose their exports as named variants next to a `DEFAULT` alias: + +```ts +models.classification.EFFICIENTNET_V2_S.DEFAULT; // resolved for this device +models.classification.EFFICIENTNET_V2_S.COREML_FP16; // always this export +models.classification.EFFICIENTNET_V2_S.XNNPACK_INT8; +``` + +`DEFAULT` is not a fixed file. It is resolved when the library loads, to the fastest export the device can actually run: + +- **iOS device** — the Core ML export where one exists, so the model reaches the Neural Engine. +- **Android, and the iOS simulator** — the XNNPACK export. The simulator has no Neural Engine and cannot run Core ML models at all. +- **Either, narrowed by your config** — only backends your app downloaded are considered. Trimming `coreml` out of an iOS build moves every `DEFAULT` back to XNNPACK rather than failing to load. + +MLX and Vulkan are never picked automatically: they win on some models and lose on others, so reach for `MLX_*` / `VULKAN_*` explicitly when you have benchmarked your case. Naming any variant directly always overrides the resolution. ### Platform notes diff --git a/packages/react-native-executorch/__tests__/api/modelVariants.test.ts b/packages/react-native-executorch/__tests__/api/modelVariants.test.ts new file mode 100644 index 0000000000..e600396585 --- /dev/null +++ b/packages/react-native-executorch/__tests__/api/modelVariants.test.ts @@ -0,0 +1,325 @@ +/** + * How the `DEFAULT` alias in the `models` registry is chosen. + * + * The alias is resolved once, when the registry module is first imported, from + * the platform and the backends the native binary was linked with. Every case + * below therefore reloads the registry behind a `jest.resetModules()` rather + * than reading the copy the test file imported. + */ +import { Platform } from 'react-native'; + +import { fakeJsi } from '../support/fakeJsi'; + +type Node = Record; + +const isObject = (value: unknown): value is Node => + typeof value === 'object' && value !== null && !Array.isArray(value); + +const isConfig = (value: unknown): value is Node => + isObject(value) && + (typeof value.modelPath === 'string' || + (isObject(value.modelPaths) && + Object.values(value.modelPaths).every((path) => typeof path === 'string'))); + +/** The `.pte` file(s) a config names, as one comparable string. */ +const modelPathsOf = (config: Node): string => + typeof config.modelPath === 'string' + ? config.modelPath + : Object.values(config.modelPaths as Node) + .map(String) + .sort() + .join('|'); + +const isUpperKey = (key: string) => /^[A-Z0-9_]+$/.test(key); + +/** Every group that names a `DEFAULT`, with the dotted path it sits at. */ +function variantGroups(node: unknown, path: string[] = []): { label: string; group: Node }[] { + if (!isObject(node)) return []; + const here = isConfig(node.DEFAULT) ? [{ label: path.join('.'), group: node }] : []; + const nested = Object.entries(node) + .filter(([key, value]) => key !== 'DEFAULT' && isObject(value)) + .flatMap(([key, value]) => variantGroups(value, [...path, key])); + return [...here, ...nested]; +} + +/** The backend variants a group lists directly, by key. */ +const namedVariants = (group: Node): [string, Node][] => + Object.entries(group).filter( + ([key, value]) => key !== 'DEFAULT' && isUpperKey(key) && isConfig(value) + ) as [string, Node][]; + +/** The variant key a group's `DEFAULT` points at, when it points at its own. */ +const defaultKeyOf = (group: Node): string | undefined => + namedVariants(group).find( + ([, value]) => modelPathsOf(value) === modelPathsOf(group.DEFAULT as Node) + )?.[0]; + +const originalOs = Platform.OS; + +/** + * Points `Platform.OS` at a target, on the module instance the next `require` + * will resolve to. Has to run after `jest.resetModules()`, which hands out a + * fresh `react-native` module. + * @param os The platform to pretend to run on. + */ +function setPlatform(os: 'ios' | 'android'): void { + (require('react-native').Platform as { OS: string }).OS = os; +} + +/** + * Reloads the registry as it would resolve on a given device. + * @param options The platform, the linked backends, and whether the device is + * a simulator. + * @returns The freshly resolved `models` registry. + */ +function registryFor(options: { + os: 'ios' | 'android'; + backends?: string[]; + isEmulator?: boolean; +}): Node { + fakeJsi.setRegisteredBackends( + options.backends ?? ['XnnpackBackend', 'CoreMLBackend', 'MLXBackend', 'VulkanBackend'] + ); + fakeJsi.setIsEmulator(options.isEmulator ?? false); + + jest.resetModules(); + setPlatform(options.os); + return require('../../src/models').models as Node; +} + +/** Every group of the reloaded registry, paired with the key it defaulted to. */ +const defaultsOf = (registry: Node) => + variantGroups(registry).map(({ label, group }) => ({ + label, + key: defaultKeyOf(group), + path: modelPathsOf(group.DEFAULT as Node), + offers: namedVariants(group).map(([key]) => key), + })); + +afterEach(() => { + jest.resetModules(); + setPlatform(originalOs as 'ios' | 'android'); +}); + +describe('DEFAULT variant resolution', () => { + it('finds groups to check', () => { + expect(defaultsOf(registryFor({ os: 'ios' })).length).toBeGreaterThan(100); + }); + + it('defaults to Core ML on iOS wherever a Core ML export exists', () => { + const offenders = defaultsOf(registryFor({ os: 'ios' })) + .filter(({ offers }) => offers.some((key) => key.startsWith('COREML'))) + .filter(({ key }) => key !== undefined && !key.startsWith('COREML')) + .map(({ label, key }) => `${label}: ${key}`); + + expect(offenders).toEqual([]); + }); + + it('never defaults to an iOS-only backend on Android', () => { + const offenders = defaultsOf(registryFor({ os: 'android' })) + .filter(({ path }) => /\/(coreml|mlx)\//.test(path)) + .map(({ label, path }) => `${label}: ${path}`); + + expect(offenders).toEqual([]); + }); + + it('defaults to XNNPACK on Android wherever an XNNPACK export exists', () => { + const offenders = defaultsOf(registryFor({ os: 'android' })) + .filter(({ offers }) => offers.some((key) => key.startsWith('XNNPACK'))) + .filter(({ key }) => key !== undefined && !key.startsWith('XNNPACK')) + .map(({ label, key }) => `${label}: ${key}`); + + expect(offenders).toEqual([]); + }); + + it('leaves Vulkan and MLX as an explicit opt-in on both platforms', () => { + for (const os of ['ios', 'android'] as const) { + const offenders = defaultsOf(registryFor({ os })) + .filter(({ key }) => key?.startsWith('VULKAN') || key?.startsWith('MLX')) + .map(({ label, key }) => `${os} ${label}: ${key}`); + + expect(offenders).toEqual([]); + } + }); + + it('falls back to XNNPACK on the iOS simulator, which cannot run Core ML', () => { + const offenders = defaultsOf(registryFor({ os: 'ios', isEmulator: true })) + .filter(({ path }) => /\/(coreml|mlx)\//.test(path)) + .map(({ label, path }) => `${label}: ${path}`); + + expect(offenders).toEqual([]); + }); + + it('falls back to XNNPACK on iOS when the app links XNNPACK only', () => { + const offenders = defaultsOf(registryFor({ os: 'ios', backends: ['XnnpackBackend'] })) + .filter(({ key }) => key !== undefined && !key.startsWith('XNNPACK')) + .map(({ label, key }) => `${label}: ${key}`); + + expect(offenders).toEqual([]); + }); + + it('picks a variant the group actually offers, on every device', () => { + const devices = [ + { os: 'ios' as const }, + { os: 'ios' as const, isEmulator: true }, + { os: 'ios' as const, backends: ['XnnpackBackend'] }, + { os: 'android' as const }, + // An app that opted out of every backend its models were published for: + // the registry still has to name a model rather than yield `undefined`. + { os: 'android' as const, backends: ['CoreMLBackend'] }, + ]; + + for (const device of devices) { + const offenders = variantGroups(registryFor(device)) + .filter(({ group }) => namedVariants(group).length > 0) + .filter(({ group }) => defaultKeyOf(group) === undefined) + .map(({ label }) => `${JSON.stringify(device)} ${label}`); + + expect(offenders).toEqual([]); + } + }); + + it('keeps the same model files reachable through named variants on both platforms', () => { + // Only the default moves per platform; the catalogue itself must not. + const keysOf = (registry: Node) => + defaultsOf(registry) + .map(({ label, offers }) => `${label}: ${offers.join(',')}`) + .sort(); + + expect(keysOf(registryFor({ os: 'ios' }))).toEqual(keysOf(registryFor({ os: 'android' }))); + }); + + it('resolves a family to the default of its first sub-group', () => { + const registry = registryFor({ os: 'ios' }); + const objectDetection = registry.objectDetection as Node; + const yolo = objectDetection.YOLO26 as Node; + const nano = yolo.NANO as Node; + + expect(yolo.DEFAULT).toBe(nano.DEFAULT); + expect(nano.DEFAULT).toBe((nano.SIZE_384 as Node).DEFAULT); + }); +}); + +describe('variants()', () => { + const A = { modelPath: 'a.pte' }; + const B = { modelPath: 'b.pte' }; + const C = { modelPath: 'c.pte' }; + + /** + * Reloads the helper module for a given device. + * @param os The platform to resolve for. + * @param backends The backends the binary reports as linked. + * @returns The freshly loaded `modelVariants` module. + */ + function load(os: 'ios' | 'android', backends?: string[]) { + fakeJsi.setRegisteredBackends(backends ?? ['XnnpackBackend', 'CoreMLBackend']); + jest.resetModules(); + setPlatform(os); + return require('../../src/modelVariants') as typeof import('../../src/modelVariants'); + } + + it('breaks a tie within one backend by declaration order', () => { + const { variants } = load('android'); + expect(variants({ XNNPACK_INT8: A, XNNPACK_FP32: B }).DEFAULT).toBe(A); + expect(variants({ XNNPACK_FP32: B, XNNPACK_INT8: A }).DEFAULT).toBe(B); + }); + + it('honours a pinned variant over the backend order', () => { + const { variants } = load('ios'); + const group = variants({ XNNPACK_FP32: A, COREML_FP16: B }, { ios: 'XNNPACK_FP32' }); + expect(group.DEFAULT).toBe(A); + }); + + it('ignores a pin whose backend the app did not link in', () => { + const { variants } = load('ios', ['XnnpackBackend']); + const group = variants({ XNNPACK_FP32: A, COREML_FP16: B }, { ios: 'COREML_FP16' }); + expect(group.DEFAULT).toBe(A); + }); + + it('applies a pin only on the platform it names', () => { + const pinned = { android: 'XNNPACK_INT8' } as const; + expect(load('android').variants({ XNNPACK_FP32: A, XNNPACK_INT8: B }, pinned).DEFAULT).toBe(B); + expect(load('ios').variants({ XNNPACK_FP32: A, XNNPACK_INT8: B }, pinned).DEFAULT).toBe(A); + }); + + it('falls back to the first variant when no preferred backend is linked in', () => { + const { variants } = load('android', ['CoreMLBackend']); + expect(variants({ COREML_FP16: C, XNNPACK_FP32: A }).DEFAULT).toBe(C); + }); + + it('keeps every named variant alongside the default', () => { + const { variants } = load('ios'); + const group = variants({ XNNPACK_FP32: A, COREML_FP16: B }); + expect(group).toEqual({ XNNPACK_FP32: A, COREML_FP16: B, DEFAULT: B }); + }); +}); + +describe('feature map', () => { + // `models...DEFAULT` only reaches the accelerated export when + // the app downloaded that backend, and `features` is the documented way to + // say which backends an app needs. A family whose feature entry is missing a + // backend it publishes therefore falls back to XNNPACK forever, quietly. + // + // Categories map to feature names one-to-one except where noted; a category + // added without an entry here fails the coverage case below. + const FEATURE_OF_CATEGORY: Record = { + classification: 'classification', + styleTransfer: 'styleTransfer', + semanticSegmentation: 'semanticSegmentation', + objectDetection: 'objectDetection', + keypointDetection: 'keypointDetection', + instanceSegmentation: 'instanceSegmentation', + voiceActivityDetection: 'vad', + speechToText: 'speechToText', + tokenizer: 'tokenizer', + llm: 'llm', + textEmbeddings: 'textEmbeddings', + privacyFilter: 'privacyFilter', + imageEmbeddings: 'imageEmbeddings', + textToImage: 'textToImage', + textToSpeech: 'textToSpeech', + ocr: 'ocr', + }; + + const { FEATURE_MAP } = require('../../scripts/download-libs.js'); + + /** Every backend folder the URLs under a registry category point into. */ + function publishedBackends(node: unknown): Set { + const found = new Set(); + const walk = (value: unknown): void => { + if (typeof value === 'string') { + const match = value.match(/\/(xnnpack|coreml|mlx|vulkan)\//); + if (match) found.add(match[1]!); + } else if (Array.isArray(value)) value.forEach(walk); + else if (isObject(value)) Object.values(value).forEach(walk); + }; + walk(node); + return found; + } + + const registry = registryFor({ os: 'ios' }); + + it('names a feature for every registry category', () => { + expect(Object.keys(registry).filter((category) => !FEATURE_OF_CATEGORY[category])).toEqual([]); + }); + + it('provisions every backend the registry publishes for that feature', () => { + const offenders: string[] = []; + + for (const [category, node] of Object.entries(registry)) { + const feature = FEATURE_OF_CATEGORY[category]!; + // Multimodal LLMs are split into their own feature; both entries cover + // the `llm` category, so the union of the two is what an LLM app gets. + const provisioned = new Set([ + ...FEATURE_MAP[feature].backends, + ...(feature === 'llm' ? FEATURE_MAP.multimodalLLM.backends : []), + ]); + + for (const backend of publishedBackends(node)) { + if (!provisioned.has(backend)) offenders.push(`${feature} is missing ${backend}`); + } + } + + expect(offenders.sort()).toEqual([]); + }); +}); diff --git a/packages/react-native-executorch/scripts/download-libs.js b/packages/react-native-executorch/scripts/download-libs.js index 459cdcfba9..4e6b75af7a 100644 --- a/packages/react-native-executorch/scripts/download-libs.js +++ b/packages/react-native-executorch/scripts/download-libs.js @@ -115,7 +115,11 @@ const ALL_LIBS = ['opencv', 'phonemis']; // features -> { backends, libs } // Backend lists are the union of what at least one model in that family ships // today (per src/models.ts). When a new variant lands for a model that adds -// e.g. coreml or vulkan support, bump the family here. +// e.g. coreml or vulkan support, bump the family here. Leaving one out is not +// only a missed optimization: `models...DEFAULT` resolves to the +// fastest export the device can actually run, so a backend that never gets +// downloaded silently drops that model back to XNNPACK. The registry test +// `feature map` in __tests__/api/modelVariants.test.ts holds this in sync. const FEATURE_MAP = { // Text-only LLMs ship xnnpack + mlx (Gemma 4 ships an MLX iOS export). llm: { backends: ['xnnpack', 'mlx'], libs: [] }, @@ -124,15 +128,16 @@ const FEATURE_MAP = { multimodalLLM: { backends: ['xnnpack', 'mlx', 'vulkan'], libs: ['opencv'] }, // Privacy filter classifiers ship xnnpack + an MLX iOS export. privacyFilter: { backends: ['xnnpack', 'mlx'], libs: [] }, - // Whisper ships xnnpack + coreml. - speechToText: { backends: ['xnnpack', 'coreml'], libs: [] }, - // Kokoro ships xnnpack only. - textToSpeech: { backends: ['xnnpack'], libs: ['phonemis'] }, + // Whisper ships xnnpack, coreml and an MLX iOS export. + speechToText: { backends: ['xnnpack', 'coreml', 'mlx'], libs: [] }, + // Kokoro ships xnnpack; Supertonic adds an MLX iOS export. + textToSpeech: { backends: ['xnnpack', 'mlx'], libs: ['phonemis'] }, // FSMN VAD — xnnpack only. vad: { backends: ['xnnpack'], libs: [] }, // LFM2.5-Embedding ships an MLX iOS export alongside xnnpack. textEmbeddings: { backends: ['xnnpack', 'mlx'], libs: [] }, - imageEmbeddings: { backends: ['xnnpack'], libs: ['opencv'] }, + // CLIP's vision encoder ships xnnpack, coreml and mlx. + imageEmbeddings: { backends: ['xnnpack', 'coreml', 'mlx'], libs: ['opencv'] }, // EfficientNet ships xnnpack + coreml. classification: { backends: ['xnnpack', 'coreml'], libs: ['opencv'] }, // YOLO is xnnpack-only, ssdlite/rf_detr add coreml → union. @@ -141,8 +146,8 @@ const FEATURE_MAP = { // keypoint adds coreml + mlx → union. (Named to track the useKeypointDetector // hook; main calls this poseEstimation.) keypointDetection: { backends: ['xnnpack', 'coreml', 'mlx'], libs: ['opencv'] }, - // DeepLab/FCN/LR-ASPP/selfie — xnnpack only. - semanticSegmentation: { backends: ['xnnpack'], libs: ['opencv'] }, + // DeepLab/FCN/LR-ASPP/selfie all ship xnnpack + coreml. + semanticSegmentation: { backends: ['xnnpack', 'coreml'], libs: ['opencv'] }, // YOLO-seg xnnpack-only, rf_detr-seg/fastsam add coreml → union. instanceSegmentation: { backends: ['xnnpack', 'coreml'], libs: ['opencv'] }, // PP-OCRv6 (DBNet + SVTR) ships xnnpack, coreml and vulkan → union. @@ -150,8 +155,8 @@ const FEATURE_MAP = { verticalOCR: { backends: ['xnnpack'], libs: ['opencv'] }, // All style-transfer presets ship xnnpack + coreml. styleTransfer: { backends: ['xnnpack', 'coreml'], libs: ['opencv'] }, - // BK-SDM — xnnpack only. - textToImage: { backends: ['xnnpack'], libs: ['opencv'] }, + // SDXS ships xnnpack + coreml. + textToImage: { backends: ['xnnpack', 'coreml'], libs: ['opencv'] }, // FastSAM ships xnnpack + coreml. segmentAnything: { backends: ['xnnpack', 'coreml'], libs: ['opencv'] }, // Tokenizer is pure-CPU string ops resolved from libexecutorch; needs no @@ -457,8 +462,12 @@ async function main() { console.log('[react-native-executorch] Native libs ready.'); } -main().catch((err) => { - console.error('[react-native-executorch] Failed to download native libs:', err.message); - console.error(' You can set RNET_SKIP_DOWNLOAD=1 to skip and provide libs manually.'); - process.exit(1); -}); +if (require.main === module) { + main().catch((err) => { + console.error('[react-native-executorch] Failed to download native libs:', err.message); + console.error(' You can set RNET_SKIP_DOWNLOAD=1 to skip and provide libs manually.'); + process.exit(1); + }); +} + +module.exports = { ALL_BACKENDS, ALL_LIBS, FEATURE_MAP }; diff --git a/packages/react-native-executorch/src/modelVariants.ts b/packages/react-native-executorch/src/modelVariants.ts new file mode 100644 index 0000000000..7f82d3fddb --- /dev/null +++ b/packages/react-native-executorch/src/modelVariants.ts @@ -0,0 +1,168 @@ +/** + * Platform-aware resolution of the `DEFAULT` alias in the {@link models} + * registry. + * + * Every model in the registry that ships more than one export lists its + * variants under backend-tagged keys (`XNNPACK_INT8`, `COREML_FP16`, ...) and + * exposes a `DEFAULT` alias next to them. Pinning that alias to one fixed + * variant means most users silently run the universal XNNPACK build on + * hardware that has a much faster accelerator sitting idle — Core ML on iOS in + * particular. `DEFAULT` is therefore resolved here, once at import time, from: + * + * 1. the platform the app is running on, + * 2. the backends actually linked into the binary — the install-time + * `react-native-executorch` block in the app's `package.json` decides these, + * and every backend is on unless the app opts out, + * 3. the order the variants are declared in, which breaks ties within a + * backend (`XNNPACK_INT8` before `XNNPACK_FP32` means "int8 unless told + * otherwise"). + * + * A model whose best variant does not follow from that ordering can pin one + * per platform — see the second argument of {@link variants}. + * + * Backends the resolver never picks on its own — MLX on iOS, Vulkan on + * Android — stay reachable through their explicit keys. They win on some + * models and lose on others, so they are an opt-in rather than a default. + * @module ModelVariants + * @internal + */ + +import { Platform } from 'react-native'; + +import { rnexecutorchJsi } from './native/bridge'; + +/** The backend prefix a variant key starts with. */ +type BackendTag = 'XNNPACK' | 'COREML' | 'MLX' | 'VULKAN'; + +/** The platforms the registry resolves defaults for. */ +type TargetPlatform = 'ios' | 'android'; + +/** + * Backends to try, best first, per platform. + * + * Core ML leads on iOS: it reaches the Neural Engine, which beats XNNPACK's + * CPU kernels on every model family that ships both. XNNPACK leads on Android + * and backs iOS up, because it is the one backend every model exports to. + */ +const BACKEND_ORDER: Record = { + ios: ['COREML', 'XNNPACK'], + android: ['XNNPACK'], +}; + +/** Variant keys pinned per platform, overriding {@link BACKEND_ORDER}. */ +type PinnedVariants = Partial>>; + +const ALL_TAGS: readonly BackendTag[] = ['XNNPACK', 'COREML', 'MLX', 'VULKAN']; + +/** + * Narrows `Platform.OS` to the platforms the registry distinguishes. + * @returns The platform to resolve defaults for. + */ +function currentPlatform(): TargetPlatform { + return Platform.OS === 'ios' ? 'ios' : 'android'; +} + +/** + * Backends that are linked into this binary and usable on this device. + * @returns The usable backend tags — every one of them when the native runtime + * cannot be asked, so that a missing answer widens the choice rather than + * narrowing it to nothing. + */ +function usableBackends(): ReadonlySet { + let registered: readonly string[] = []; + try { + registered = rnexecutorchJsi.getExecuTorchRegisteredBackends(); + } catch { + registered = []; + } + if (registered.length === 0) return new Set(ALL_TAGS); + + const names = registered.map((name) => name.toLowerCase()); + const usable = ALL_TAGS.filter((tag) => names.some((name) => name.startsWith(tag.toLowerCase()))); + + // The simulator links the Core ML backend but cannot run it: it has no + // Neural Engine, and MPSGraph refuses the compiled models outright. MLX only + // ever ships a device slice, so it drops out of `registered` on its own. + if (currentPlatform() === 'ios' && rnexecutorchJsi.isEmulator === true) { + return new Set(usable.filter((tag) => tag !== 'COREML' && tag !== 'MLX')); + } + return new Set(usable); +} + +const PLATFORM = currentPlatform(); +const USABLE = usableBackends(); + +/** + * Reads the backend out of a variant key. + * @param key The variant key, e.g. `COREML_FP16`. + * @returns The backend the key names, or `undefined` when it names none. + */ +function backendOf(key: string): BackendTag | undefined { + return ALL_TAGS.find((tag) => key === tag || key.startsWith(`${tag}_`)); +} + +/** + * Picks the variant key this platform should default to. + * @param keys The group's variant keys, in declaration order. + * @param pinned Per-platform overrides. + * @returns The chosen key. + */ +function pickVariant( + keys: readonly string[], + pinned?: PinnedVariants> +): string { + const pin = pinned?.[PLATFORM]; + const pinnedBackend = pin === undefined ? undefined : backendOf(pin); + if (pin !== undefined && keys.includes(pin) && pinnedBackend && USABLE.has(pinnedBackend)) { + return pin; + } + + for (const tag of BACKEND_ORDER[PLATFORM]) { + if (!USABLE.has(tag)) continue; + const match = keys.find((key) => backendOf(key) === tag); + if (match !== undefined) return match; + } + + // No preferred backend is both published for this model and linked into the + // build — an app that opted out of the backends its models need. Hand back + // the first variant so the registry still names a model and the failure + // surfaces at load, where the error says which backend is missing. + return keys[0]!; +} + +/** + * Adds a platform-resolved `DEFAULT` to a group of backend variants. + * + * Declare the variants best-first within each backend: with several exports + * from the same backend, the earliest one wins. + * @typeParam V The variant map. + * @param map The group's variants, keyed by backend and precision. + * @param pinned Variant keys to prefer on a given platform, for models whose + * best export does not follow from the declaration order. Ignored when the + * pinned variant's backend is not linked into the build. + * @returns The variants, plus the `DEFAULT` alias for this platform. + */ +export function variants>( + map: V, + pinned?: PinnedVariants +): V & { readonly DEFAULT: V[keyof V] } { + const key = pickVariant(Object.keys(map), pinned); + return { ...map, DEFAULT: map[key] as V[keyof V] }; +} + +/** + * Adds a `DEFAULT` to a group of sub-groups — a model family split by scale or + * input size — mirroring the `DEFAULT` of the first sub-group declared. + * + * The sub-group resolved its own default per platform, so the family inherits + * that without repeating the rules. + * @typeParam V The sub-group map. + * @param map The family's sub-groups, most representative first. + * @returns The sub-groups, plus the inherited `DEFAULT`. + */ +export function family>( + map: V +): V & { readonly DEFAULT: V[keyof V]['DEFAULT'] } { + const first = Object.keys(map)[0]!; + return { ...map, DEFAULT: map[first]!.DEFAULT as V[keyof V]['DEFAULT'] }; +} diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts index 1985444133..b2a52d74d2 100644 --- a/packages/react-native-executorch/src/models.ts +++ b/packages/react-native-executorch/src/models.ts @@ -5,6 +5,12 @@ * vision, speech synthesis/recognition, natural language processing, and large * language models (LLMs). Each entry includes verified remote `.pte` download * URLs, tokenizer/phonemizer files, preprocessing parameters, and label maps. + * + * A model that ships several exports lists them under backend-tagged keys and + * wraps the group in `variants`, which adds the `DEFAULT` alias that resolves + * to the fastest export the current platform can run — see `modelVariants.ts`. + * Within one backend the first variant declared wins, so keep the group + * ordered best-first. * @module Models */ @@ -28,6 +34,7 @@ import { } from './extensions/speech/tasks/whisperSpeechToText'; import type { PaddleOcrModel } from './extensions/cv/tasks/paddleOcr'; import type { LLMModel } from './extensions/llm/tasks/llmChatSession'; +import { variants, family } from './modelVariants'; import { IMAGENET_NORM, IMAGENET1K_LABELS, @@ -1333,6 +1340,12 @@ const QWEN3_4B_XNNPACK_BF16: LLMModel = { * This provides Hugging Face repository URLs and baseline configurations for * tasks, allowing quick model loading and execution without manual option * setup. + * + * Models published for more than one backend expose their exports as named + * variants (`XNNPACK_INT8`, `COREML_FP16`, ...) plus a `DEFAULT` alias. The + * alias is chosen for the device the app runs on: Core ML on iOS hardware, + * XNNPACK on Android and on the iOS simulator, always narrowed to the backends + * the app actually linked in. Reach for a named variant to override that. * @category Models */ export const models = { @@ -1347,12 +1360,11 @@ export const models = { * architecture providing high accuracy for general-purpose image * classification. */ - EFFICIENTNET_V2_S: { - DEFAULT: EFFICIENTNET_V2_S_XNNPACK_INT8, + EFFICIENTNET_V2_S: variants({ XNNPACK_INT8: EFFICIENTNET_V2_S_XNNPACK_INT8, XNNPACK_FP32: EFFICIENTNET_V2_S_XNNPACK_FP32, COREML_FP16: EFFICIENTNET_V2_S_COREML_FP16, - }, + }), }, /** @@ -1364,42 +1376,38 @@ export const models = { * Fast neural style transfer model generating a vibrant, artistic "Candy" * style effect. */ - CANDY: { - DEFAULT: STYLE_TRANSFER_CANDY_XNNPACK_INT8, - XNNPACK_FP32: STYLE_TRANSFER_CANDY_XNNPACK_FP32, + CANDY: variants({ XNNPACK_INT8: STYLE_TRANSFER_CANDY_XNNPACK_INT8, + XNNPACK_FP32: STYLE_TRANSFER_CANDY_XNNPACK_FP32, COREML_FP16: STYLE_TRANSFER_CANDY_COREML_FP16, - }, + }), /** * Fast neural style transfer model applying a classic tile mosaic artistic * pattern. */ - MOSAIC: { - DEFAULT: STYLE_TRANSFER_MOSAIC_XNNPACK_INT8, - XNNPACK_FP32: STYLE_TRANSFER_MOSAIC_XNNPACK_FP32, + MOSAIC: variants({ XNNPACK_INT8: STYLE_TRANSFER_MOSAIC_XNNPACK_INT8, + XNNPACK_FP32: STYLE_TRANSFER_MOSAIC_XNNPACK_FP32, COREML_FP16: STYLE_TRANSFER_MOSAIC_COREML_FP16, - }, + }), /** * Fast neural style transfer model applying a painterly "Rain Princess" oil * painting aesthetic. */ - RAIN_PRINCESS: { - DEFAULT: STYLE_TRANSFER_RAIN_PRINCESS_XNNPACK_INT8, - XNNPACK_FP32: STYLE_TRANSFER_RAIN_PRINCESS_XNNPACK_FP32, + RAIN_PRINCESS: variants({ XNNPACK_INT8: STYLE_TRANSFER_RAIN_PRINCESS_XNNPACK_INT8, + XNNPACK_FP32: STYLE_TRANSFER_RAIN_PRINCESS_XNNPACK_FP32, COREML_FP16: STYLE_TRANSFER_RAIN_PRINCESS_COREML_FP16, - }, + }), /** * Fast neural style transfer model applying Francis Picabia's "Udnie" * abstract art style. */ - UDNIE: { - DEFAULT: STYLE_TRANSFER_UDNIE_XNNPACK_INT8, - XNNPACK_FP32: STYLE_TRANSFER_UDNIE_XNNPACK_FP32, + UDNIE: variants({ XNNPACK_INT8: STYLE_TRANSFER_UDNIE_XNNPACK_INT8, + XNNPACK_FP32: STYLE_TRANSFER_UDNIE_XNNPACK_FP32, COREML_FP16: STYLE_TRANSFER_UDNIE_COREML_FP16, - }, + }), }, /** @@ -1412,84 +1420,76 @@ export const models = { * background separation. Categorizes pixels into `background` and `person`. * Ideal for background blur and replacement effects. */ - SELFIE_SEGMENTATION: { - DEFAULT: SELFIE_SEGMENTATION_XNNPACK_FP32, + SELFIE_SEGMENTATION: variants({ XNNPACK_FP32: SELFIE_SEGMENTATION_XNNPACK_FP32, COREML_FP16: SELFIE_SEGMENTATION_COREML_FP16, - }, + }), /** * MediaPipe Selfie Segmentation, landscape orientation. A separate * 256x144 checkpoint rather than a resize of the portrait model. */ - SELFIE_SEGMENTATION_LANDSCAPE: { - DEFAULT: SELFIE_SEGMENTATION_LANDSCAPE_XNNPACK_FP32, + SELFIE_SEGMENTATION_LANDSCAPE: variants({ XNNPACK_FP32: SELFIE_SEGMENTATION_LANDSCAPE_XNNPACK_FP32, COREML_FP16: SELFIE_SEGMENTATION_LANDSCAPE_COREML_FP16, - }, + }), /** * Lite R-ASPP semantic segmentation model with MobileNetV3-Large backbone * (21 classes, see {@link PASCAL_VOC_LABELS}). Optimized for low-latency, * real-time pixel-level segmentation on mobile devices. */ - LRASPP_MOBILENET_V3_LARGE: { - DEFAULT: LRASPP_MOBILENET_V3_LARGE_XNNPACK_INT8, - XNNPACK_FP32: LRASPP_MOBILENET_V3_LARGE_XNNPACK_FP32, + LRASPP_MOBILENET_V3_LARGE: variants({ XNNPACK_INT8: LRASPP_MOBILENET_V3_LARGE_XNNPACK_INT8, + XNNPACK_FP32: LRASPP_MOBILENET_V3_LARGE_XNNPACK_FP32, COREML_FP16: LRASPP_MOBILENET_V3_LARGE_COREML_FP16, - }, + }), /** * DeepLabV3 semantic segmentation model with ResNet-50 backbone (21 * classes, see {@link PASCAL_VOC_LABELS}). High-accuracy segmentation * utilizing atrous spatial pyramid pooling. */ - DEEPLAB_V3_RESNET50: { - DEFAULT: DEEPLAB_V3_RESNET50_XNNPACK_INT8, - XNNPACK_FP32: DEEPLAB_V3_RESNET50_XNNPACK_FP32, + DEEPLAB_V3_RESNET50: variants({ XNNPACK_INT8: DEEPLAB_V3_RESNET50_XNNPACK_INT8, + XNNPACK_FP32: DEEPLAB_V3_RESNET50_XNNPACK_FP32, COREML_FP16: DEEPLAB_V3_RESNET50_COREML_FP16, - }, + }), /** * DeepLabV3 semantic segmentation model with ResNet-101 backbone (21 * classes, see {@link PASCAL_VOC_LABELS}). High-capacity backbone for * maximum segmentation detail and boundary accuracy. */ - DEEPLAB_V3_RESNET101: { - DEFAULT: DEEPLAB_V3_RESNET101_XNNPACK_INT8, - XNNPACK_FP32: DEEPLAB_V3_RESNET101_XNNPACK_FP32, + DEEPLAB_V3_RESNET101: variants({ XNNPACK_INT8: DEEPLAB_V3_RESNET101_XNNPACK_INT8, + XNNPACK_FP32: DEEPLAB_V3_RESNET101_XNNPACK_FP32, COREML_FP16: DEEPLAB_V3_RESNET101_COREML_FP16, - }, + }), /** * DeepLabV3 semantic segmentation model with MobileNetV3-Large backbone (21 * classes, see {@link PASCAL_VOC_LABELS}). Combines DeepLabV3 feature * extraction quality with a lightweight mobile backbone. */ - DEEPLAB_V3_MOBILENET_V3_LARGE: { - DEFAULT: DEEPLAB_V3_MOBILENET_V3_LARGE_XNNPACK_INT8, - XNNPACK_FP32: DEEPLAB_V3_MOBILENET_V3_LARGE_XNNPACK_FP32, + DEEPLAB_V3_MOBILENET_V3_LARGE: variants({ XNNPACK_INT8: DEEPLAB_V3_MOBILENET_V3_LARGE_XNNPACK_INT8, + XNNPACK_FP32: DEEPLAB_V3_MOBILENET_V3_LARGE_XNNPACK_FP32, COREML_FP16: DEEPLAB_V3_MOBILENET_V3_LARGE_COREML_FP16, - }, + }), /** * Fully Convolutional Network (FCN) semantic segmentation model with * ResNet-50 backbone (21 classes, see {@link PASCAL_VOC_LABELS}). */ - FCN_RESNET50: { - DEFAULT: FCN_RESNET50_XNNPACK_INT8, - XNNPACK_FP32: FCN_RESNET50_XNNPACK_FP32, + FCN_RESNET50: variants({ XNNPACK_INT8: FCN_RESNET50_XNNPACK_INT8, + XNNPACK_FP32: FCN_RESNET50_XNNPACK_FP32, COREML_FP16: FCN_RESNET50_COREML_FP16, - }, + }), /** * Fully Convolutional Network (FCN) semantic segmentation model with * ResNet-101 backbone (21 classes, see {@link PASCAL_VOC_LABELS}). */ - FCN_RESNET101: { - DEFAULT: FCN_RESNET101_XNNPACK_INT8, - XNNPACK_FP32: FCN_RESNET101_XNNPACK_FP32, + FCN_RESNET101: variants({ XNNPACK_INT8: FCN_RESNET101_XNNPACK_INT8, + XNNPACK_FP32: FCN_RESNET101_XNNPACK_FP32, COREML_FP16: FCN_RESNET101_COREML_FP16, - }, + }), }, /** @@ -1501,139 +1501,116 @@ export const models = { * (see {@link COCO_CLASSES}) at 320x320 resolution. Fast, lightweight * detector suited for real-time mobile applications. */ - SSDLITE320_MOBILENET_V3_LARGE: { - DEFAULT: SSDLITE320_MOBILENET_V3_LARGE_XNNPACK_FP32, + SSDLITE320_MOBILENET_V3_LARGE: variants({ XNNPACK_FP32: SSDLITE320_MOBILENET_V3_LARGE_XNNPACK_FP32, COREML_FP16: SSDLITE320_MOBILENET_V3_LARGE_COREML_FP16, - }, + }), /** * RF-DETR (Roboflow Detection Transformer) Nano variant trained on COCO * (see {@link COCO_CLASSES}). Modern end-to-end DINOv2-based transformer * object detector. */ - RFDETR_NANO: { - DEFAULT: RFDETR_NANO_DETECTOR_XNNPACK_FP32, + RFDETR_NANO: variants({ XNNPACK_FP32: RFDETR_NANO_DETECTOR_XNNPACK_FP32, COREML_FP16: RFDETR_NANO_DETECTOR_COREML_FP16, - }, + }), /** * Ultralytics YOLO26 real-time object detection models trained on COCO (80 * classes, see {@link COCO_CLASSES_YOLO}). Available across multiple scale * sizes (NANO, SMALL, MEDIUM, LARGE, XLARGE) and resolutions (384x384, * 512x512, 640x640). */ - YOLO26: { - DEFAULT: YOLO26_NANO_384_XNNPACK_FP32, + YOLO26: family({ /** * Nano scale YOLO26 object detection model. High speed, ultra low * latency. */ - NANO: { - DEFAULT: YOLO26_NANO_384_XNNPACK_FP32, - SIZE_384: { - DEFAULT: YOLO26_NANO_384_XNNPACK_FP32, + NANO: family({ + SIZE_384: variants({ XNNPACK_FP32: YOLO26_NANO_384_XNNPACK_FP32, COREML_FP16: YOLO26_NANO_384_COREML_FP16, - }, - SIZE_512: { - DEFAULT: YOLO26_NANO_512_XNNPACK_FP32, + }), + SIZE_512: variants({ XNNPACK_FP32: YOLO26_NANO_512_XNNPACK_FP32, COREML_FP16: YOLO26_NANO_512_COREML_FP16, - }, - SIZE_640: { - DEFAULT: YOLO26_NANO_640_XNNPACK_FP32, + }), + SIZE_640: variants({ XNNPACK_FP32: YOLO26_NANO_640_XNNPACK_FP32, COREML_FP16: YOLO26_NANO_640_COREML_FP16, - }, - }, + }), + }), /** * Small scale YOLO26 object detection model. Balanced latency and * accuracy. */ - SMALL: { - DEFAULT: YOLO26_SMALL_384_XNNPACK_FP32, - SIZE_384: { - DEFAULT: YOLO26_SMALL_384_XNNPACK_FP32, + SMALL: family({ + SIZE_384: variants({ XNNPACK_FP32: YOLO26_SMALL_384_XNNPACK_FP32, COREML_FP16: YOLO26_SMALL_384_COREML_FP16, - }, - SIZE_512: { - DEFAULT: YOLO26_SMALL_512_XNNPACK_FP32, + }), + SIZE_512: variants({ XNNPACK_FP32: YOLO26_SMALL_512_XNNPACK_FP32, COREML_FP16: YOLO26_SMALL_512_COREML_FP16, - }, - SIZE_640: { - DEFAULT: YOLO26_SMALL_640_XNNPACK_FP32, + }), + SIZE_640: variants({ XNNPACK_FP32: YOLO26_SMALL_640_XNNPACK_FP32, COREML_FP16: YOLO26_SMALL_640_COREML_FP16, - }, - }, + }), + }), /** * Medium scale YOLO26 object detection model. Higher precision for * complex scenes. */ - MEDIUM: { - DEFAULT: YOLO26_MEDIUM_384_XNNPACK_FP32, - SIZE_384: { - DEFAULT: YOLO26_MEDIUM_384_XNNPACK_FP32, + MEDIUM: family({ + SIZE_384: variants({ XNNPACK_FP32: YOLO26_MEDIUM_384_XNNPACK_FP32, COREML_FP16: YOLO26_MEDIUM_384_COREML_FP16, - }, - SIZE_512: { - DEFAULT: YOLO26_MEDIUM_512_XNNPACK_FP32, + }), + SIZE_512: variants({ XNNPACK_FP32: YOLO26_MEDIUM_512_XNNPACK_FP32, COREML_FP16: YOLO26_MEDIUM_512_COREML_FP16, - }, - SIZE_640: { - DEFAULT: YOLO26_MEDIUM_640_XNNPACK_FP32, + }), + SIZE_640: variants({ XNNPACK_FP32: YOLO26_MEDIUM_640_XNNPACK_FP32, COREML_FP16: YOLO26_MEDIUM_640_COREML_FP16, - }, - }, + }), + }), /** * Large scale YOLO26 object detection model. High accuracy model variant. */ - LARGE: { - DEFAULT: YOLO26_LARGE_384_XNNPACK_FP32, - SIZE_384: { - DEFAULT: YOLO26_LARGE_384_XNNPACK_FP32, + LARGE: family({ + SIZE_384: variants({ XNNPACK_FP32: YOLO26_LARGE_384_XNNPACK_FP32, COREML_FP16: YOLO26_LARGE_384_COREML_FP16, - }, - SIZE_512: { - DEFAULT: YOLO26_LARGE_512_XNNPACK_FP32, + }), + SIZE_512: variants({ XNNPACK_FP32: YOLO26_LARGE_512_XNNPACK_FP32, COREML_FP16: YOLO26_LARGE_512_COREML_FP16, - }, - SIZE_640: { - DEFAULT: YOLO26_LARGE_640_XNNPACK_FP32, + }), + SIZE_640: variants({ XNNPACK_FP32: YOLO26_LARGE_640_XNNPACK_FP32, COREML_FP16: YOLO26_LARGE_640_COREML_FP16, - }, - }, + }), + }), /** * Extra Large scale YOLO26 object detection model. Maximum detection * performance. */ - XLARGE: { - DEFAULT: YOLO26_XLARGE_384_XNNPACK_FP32, - SIZE_384: { - DEFAULT: YOLO26_XLARGE_384_XNNPACK_FP32, + XLARGE: family({ + SIZE_384: variants({ XNNPACK_FP32: YOLO26_XLARGE_384_XNNPACK_FP32, COREML_FP16: YOLO26_XLARGE_384_COREML_FP16, - }, - SIZE_512: { - DEFAULT: YOLO26_XLARGE_512_XNNPACK_FP32, + }), + SIZE_512: variants({ XNNPACK_FP32: YOLO26_XLARGE_512_XNNPACK_FP32, COREML_FP16: YOLO26_XLARGE_512_COREML_FP16, - }, - SIZE_640: { - DEFAULT: YOLO26_XLARGE_640_XNNPACK_FP32, + }), + SIZE_640: variants({ XNNPACK_FP32: YOLO26_XLARGE_640_XNNPACK_FP32, COREML_FP16: YOLO26_XLARGE_640_COREML_FP16, - }, - }, - }, + }), + }), + }), }, /** @@ -1646,40 +1623,34 @@ export const models = { * landmark locator (eyes, nose, mouth, ears, see * {@link BLAZEFACE_LANDMARKS}). */ - BLAZEFACE: { - DEFAULT: BLAZEFACE_XNNPACK_FP32, + BLAZEFACE: variants({ XNNPACK_FP32: BLAZEFACE_XNNPACK_FP32, - }, + }), /** * YOLO26 human pose estimation model predicting 17 COCO body keypoints (see * {@link COCO_LANDMARKS}). Available across 384x384, 512x512, and 640x640 * resolutions. */ - YOLO26_POSE: { - DEFAULT: YOLO26_POSE_384_XNNPACK_FP32, - SIZE_384: { - DEFAULT: YOLO26_POSE_384_XNNPACK_FP32, + YOLO26_POSE: family({ + SIZE_384: variants({ XNNPACK_FP32: YOLO26_POSE_384_XNNPACK_FP32, - }, - SIZE_512: { - DEFAULT: YOLO26_POSE_512_XNNPACK_FP32, + }), + SIZE_512: variants({ XNNPACK_FP32: YOLO26_POSE_512_XNNPACK_FP32, - }, - SIZE_640: { - DEFAULT: YOLO26_POSE_640_XNNPACK_FP32, + }), + SIZE_640: variants({ XNNPACK_FP32: YOLO26_POSE_640_XNNPACK_FP32, - }, - }, + }), + }), /** * RF-DETR (Roboflow Detection Transformer) pose keypoint detector * predicting 17 COCO body keypoints (see {@link COCO_LANDMARKS}). */ - RFDETR_KEYPOINT: { - DEFAULT: RFDETR_KEYPOINT_XNNPACK_FP32, + RFDETR_KEYPOINT: variants({ XNNPACK_FP32: RFDETR_KEYPOINT_XNNPACK_FP32, COREML_FP32: RFDETR_KEYPOINT_COREML_FP32, MLX_FP32: RFDETR_KEYPOINT_MLX_FP32, - }, + }), }, /** @@ -1696,134 +1667,110 @@ export const models = { /** * FastSAM Small - lightweight instance segmenter for mobile. */ - S: { - DEFAULT: FASTSAM_S_XNNPACK_FP32, + S: variants({ XNNPACK_FP32: FASTSAM_S_XNNPACK_FP32, COREML_FP16: FASTSAM_S_COREML_FP16, - }, + }), /** * FastSAM Extra Large - high-accuracy instance segmenter. */ - X: { - DEFAULT: FASTSAM_X_XNNPACK_FP32, + X: variants({ XNNPACK_FP32: FASTSAM_X_XNNPACK_FP32, COREML_FP16: FASTSAM_X_COREML_FP16, - }, + }), }, /** * RF-DETR (Roboflow Detection Transformer) Nano instance segmentation model * predicting COCO class masks and bounding boxes (see * {@link COCO_CLASSES}). */ - RFDETR_NANO: { - DEFAULT: RFDETR_NANO_SEG_COREML_FP16, - COREML_FP16: RFDETR_NANO_SEG_COREML_FP16, + RFDETR_NANO: variants({ XNNPACK_FP32: RFDETR_NANO_SEG_XNNPACK_FP32, - }, + COREML_FP16: RFDETR_NANO_SEG_COREML_FP16, + }), /** * YOLO26 instance segmentation models predicting COCO class instance masks * and bounding boxes (see {@link COCO_CLASSES_YOLO}). Available across * multiple sizes (NANO, SMALL, MEDIUM, LARGE, XLARGE) and resolutions * (384x384, 512x512, 640x640). */ - YOLO26: { - DEFAULT: YOLO26_NANO_SEG_384_XNNPACK_FP32, + YOLO26: family({ /** * Nano scale YOLO26 instance segmentation model. High speed, ultra low * latency mask generation. */ - NANO: { - DEFAULT: YOLO26_NANO_SEG_384_XNNPACK_FP32, - SIZE_384: { - DEFAULT: YOLO26_NANO_SEG_384_XNNPACK_FP32, + NANO: family({ + SIZE_384: variants({ XNNPACK_FP32: YOLO26_NANO_SEG_384_XNNPACK_FP32, - }, - SIZE_512: { - DEFAULT: YOLO26_NANO_SEG_512_XNNPACK_FP32, + }), + SIZE_512: variants({ XNNPACK_FP32: YOLO26_NANO_SEG_512_XNNPACK_FP32, - }, - SIZE_640: { - DEFAULT: YOLO26_NANO_SEG_640_XNNPACK_FP32, + }), + SIZE_640: variants({ XNNPACK_FP32: YOLO26_NANO_SEG_640_XNNPACK_FP32, - }, - }, + }), + }), /** * Small scale YOLO26 instance segmentation model. Balanced latency and * mask accuracy. */ - SMALL: { - DEFAULT: YOLO26_SMALL_SEG_384_XNNPACK_FP32, - SIZE_384: { - DEFAULT: YOLO26_SMALL_SEG_384_XNNPACK_FP32, + SMALL: family({ + SIZE_384: variants({ XNNPACK_FP32: YOLO26_SMALL_SEG_384_XNNPACK_FP32, - }, - SIZE_512: { - DEFAULT: YOLO26_SMALL_SEG_512_XNNPACK_FP32, + }), + SIZE_512: variants({ XNNPACK_FP32: YOLO26_SMALL_SEG_512_XNNPACK_FP32, - }, - SIZE_640: { - DEFAULT: YOLO26_SMALL_SEG_640_XNNPACK_FP32, + }), + SIZE_640: variants({ XNNPACK_FP32: YOLO26_SMALL_SEG_640_XNNPACK_FP32, - }, - }, + }), + }), /** * Medium scale YOLO26 instance segmentation model. Higher mask boundary * precision for complex multi-object scenes. */ - MEDIUM: { - DEFAULT: YOLO26_MEDIUM_SEG_384_XNNPACK_FP32, - SIZE_384: { - DEFAULT: YOLO26_MEDIUM_SEG_384_XNNPACK_FP32, + MEDIUM: family({ + SIZE_384: variants({ XNNPACK_FP32: YOLO26_MEDIUM_SEG_384_XNNPACK_FP32, - }, - SIZE_512: { - DEFAULT: YOLO26_MEDIUM_SEG_512_XNNPACK_FP32, + }), + SIZE_512: variants({ XNNPACK_FP32: YOLO26_MEDIUM_SEG_512_XNNPACK_FP32, - }, - SIZE_640: { - DEFAULT: YOLO26_MEDIUM_SEG_640_XNNPACK_FP32, + }), + SIZE_640: variants({ XNNPACK_FP32: YOLO26_MEDIUM_SEG_640_XNNPACK_FP32, - }, - }, + }), + }), /** * Large scale YOLO26 instance segmentation model. High accuracy instance * segmentation variant for demanding visual pipelines. */ - LARGE: { - DEFAULT: YOLO26_LARGE_SEG_384_XNNPACK_FP32, - SIZE_384: { - DEFAULT: YOLO26_LARGE_SEG_384_XNNPACK_FP32, + LARGE: family({ + SIZE_384: variants({ XNNPACK_FP32: YOLO26_LARGE_SEG_384_XNNPACK_FP32, - }, - SIZE_512: { - DEFAULT: YOLO26_LARGE_SEG_512_XNNPACK_FP32, + }), + SIZE_512: variants({ XNNPACK_FP32: YOLO26_LARGE_SEG_512_XNNPACK_FP32, - }, - SIZE_640: { - DEFAULT: YOLO26_LARGE_SEG_640_XNNPACK_FP32, + }), + SIZE_640: variants({ XNNPACK_FP32: YOLO26_LARGE_SEG_640_XNNPACK_FP32, - }, - }, + }), + }), /** * Extra Large scale YOLO26 instance segmentation model. Maximum instance * segmentation and mask delineation performance. */ - XLARGE: { - DEFAULT: YOLO26_XLARGE_SEG_384_XNNPACK_FP32, - SIZE_384: { - DEFAULT: YOLO26_XLARGE_SEG_384_XNNPACK_FP32, + XLARGE: family({ + SIZE_384: variants({ XNNPACK_FP32: YOLO26_XLARGE_SEG_384_XNNPACK_FP32, - }, - SIZE_512: { - DEFAULT: YOLO26_XLARGE_SEG_512_XNNPACK_FP32, + }), + SIZE_512: variants({ XNNPACK_FP32: YOLO26_XLARGE_SEG_512_XNNPACK_FP32, - }, - SIZE_640: { - DEFAULT: YOLO26_XLARGE_SEG_640_XNNPACK_FP32, + }), + SIZE_640: variants({ XNNPACK_FP32: YOLO26_XLARGE_SEG_640_XNNPACK_FP32, - }, - }, - }, + }), + }), + }), }, /** @@ -1836,10 +1783,9 @@ export const models = { * model. Extremely lightweight model evaluating continuous speech * probability chunks for live mic streaming and STT preprocessing. */ - FSMN_VAD: { - DEFAULT: FSMN_VAD_XNNPACK_FP32, + FSMN_VAD: variants({ XNNPACK_FP32: FSMN_VAD_XNNPACK_FP32, - }, + }), }, /** @@ -1856,67 +1802,61 @@ export const models = { * Multilingual Whisper Tiny model. Supporting 99+ languages. High speed * speech recognition. */ - TINY: { - DEFAULT: WHISPER_TINY_XNNPACK_FP32, + TINY: variants({ XNNPACK_FP32: WHISPER_TINY_XNNPACK_FP32, COREML_FP16: WHISPER_TINY_COREML_FP16, MLX_BF16: WHISPER_TINY_MLX_BF16, MLX_INT8: WHISPER_TINY_MLX_INT8, - }, + }), /** * Multilingual Whisper Base model. Higher accuracy across supported * languages. */ - BASE: { - DEFAULT: WHISPER_BASE_XNNPACK_FP32, + BASE: variants({ XNNPACK_FP32: WHISPER_BASE_XNNPACK_FP32, COREML_FP16: WHISPER_BASE_COREML_FP16, MLX_BF16: WHISPER_BASE_MLX_BF16, MLX_INT8: WHISPER_BASE_MLX_INT8, - }, + }), /** * Multilingual Whisper Small model. Best accuracy for complex * multi-language audio. */ - SMALL: { - DEFAULT: WHISPER_SMALL_XNNPACK_FP32, + SMALL: variants({ XNNPACK_FP32: WHISPER_SMALL_XNNPACK_FP32, COREML_FP16: WHISPER_SMALL_COREML_FP16, MLX_INT8: WHISPER_SMALL_MLX_INT8, - }, + }), /** English-only optimized Whisper models (`TINY`, `BASE`, `SMALL`). */ EN: { /** * English-only Whisper Tiny model. Fast and compact for English STT. */ - TINY: { - DEFAULT: WHISPER_TINY_EN_XNNPACK_FP32, + TINY: variants({ XNNPACK_FP32: WHISPER_TINY_EN_XNNPACK_FP32, COREML_FP16: WHISPER_TINY_EN_COREML_FP16, MLX_BF16: WHISPER_TINY_EN_MLX_BF16, MLX_INT8: WHISPER_TINY_EN_MLX_INT8, - }, + }), /** * English-only Whisper Base model. High accuracy English speech * recognition. */ - BASE: { - DEFAULT: WHISPER_BASE_EN_XNNPACK_FP32, + BASE: variants({ XNNPACK_FP32: WHISPER_BASE_EN_XNNPACK_FP32, COREML_FP16: WHISPER_BASE_EN_COREML_FP16, MLX_BF16: WHISPER_BASE_EN_MLX_BF16, MLX_INT8: WHISPER_BASE_EN_MLX_INT8, - }, + }), /** * English-only Whisper Small model. Superior accuracy for English * transcription. */ - SMALL: { - DEFAULT: WHISPER_SMALL_EN_XNNPACK_FP32, + SMALL: variants({ XNNPACK_FP32: WHISPER_SMALL_EN_XNNPACK_FP32, COREML_FP16: WHISPER_SMALL_EN_COREML_FP16, MLX_INT8: WHISPER_SMALL_EN_MLX_INT8, - }, + }), }, }, }, @@ -1941,58 +1881,53 @@ export const models = { * reasoning, instruction following, and fast multi-turn conversational chat * on mobile devices. */ - LFM2_5_1_2B: { - DEFAULT: LFM2_5_1_2B_XNNPACK_8DA4W, + LFM2_5_1_2B: variants({ XNNPACK_8DA4W: LFM2_5_1_2B_XNNPACK_8DA4W, XNNPACK_FP16: LFM2_5_1_2B_XNNPACK_FP16, MLX_INT4: LFM2_5_1_2B_MLX_INT4, - }, + }), /** * Liquid AI LFM 2.5 350M ultra-compact hybrid language model. Optimized for * minimal memory footprint and sub-second first-token response times. Ideal * for lightweight text completion, fast intent classification, query * routing, and low-latency chat on resource-constrained edge hardware. */ - LFM2_5_350M: { - DEFAULT: LFM2_5_350M_XNNPACK_8DA4W, + LFM2_5_350M: variants({ XNNPACK_8DA4W: LFM2_5_350M_XNNPACK_8DA4W, XNNPACK_FP16: LFM2_5_350M_XNNPACK_FP16, MLX_INT4: LFM2_5_350M_MLX_INT4, - }, + }), /** * Liquid AI LFM 2.5 VL 450M lightweight multimodal vision-language model. * Combines Liquid hybrid language modeling with visual token embeddings for * real-time on-device visual question answering (VQA), image description, * UI element inspection, and low-latency multimodal conversational agents. */ - LFM2_5_VL_450M: { - DEFAULT: LFM2_5_VL_450M_XNNPACK_8DA4W, + LFM2_5_VL_450M: variants({ XNNPACK_8DA4W: LFM2_5_VL_450M_XNNPACK_8DA4W, MLX_INT4: LFM2_5_VL_450M_MLX_INT4, VULKAN_8DA4W: LFM2_5_VL_450M_VULKAN_8DA4W, - }, + }), /** * Liquid AI LFM 2.5 VL 1.6B high-capacity vision-language model. Provides * fine-grained visual scene understanding, document/chart interpretation, * detailed image captioning, and multi-turn visual dialogue with higher * precision and reasoning fidelity than the 450M variant. */ - LFM2_5_VL_1_6B: { - DEFAULT: LFM2_5_VL_1_6B_XNNPACK_8DA4W, + LFM2_5_VL_1_6B: variants({ XNNPACK_8DA4W: LFM2_5_VL_1_6B_XNNPACK_8DA4W, VULKAN_8DA4W: LFM2_5_VL_1_6B_VULKAN_8DA4W, - }, + }), /** * Bielik v3 1.5B bilingual Polish & English language model, developed by * SpeakLeash. Fine-tuned on curated Polish corpora and instruction datasets * for native Polish cultural nuance, grammar accuracy, idioms, and * high-fidelity bidirectional Polish-English translation. */ - BIELIK_V3_1_5B: { - DEFAULT: BIELIK_V3_1_5B_XNNPACK_8DA4W, + BIELIK_V3_1_5B: variants({ XNNPACK_8DA4W: BIELIK_V3_1_5B_XNNPACK_8DA4W, XNNPACK_FP16: BIELIK_V3_1_5B_XNNPACK_FP16, - }, + }), /** * Meta Llama 3.2 1B lightweight instruction-tuned multilingual model. * Features Grouped-Query Attention (GQA) and SpinQuant quantization for @@ -2000,172 +1935,156 @@ export const models = { * text summarization, prompt rewriting, and lightweight conversational * assistance. */ - LLAMA3_2_1B: { - DEFAULT: LLAMA3_2_1B_SPINQUANT, + LLAMA3_2_1B: variants({ XNNPACK_SPINQUANT: LLAMA3_2_1B_SPINQUANT, XNNPACK_BF16: LLAMA3_2_1B_BF16, - }, + }), /** * Meta Llama 3.2 3B instruction-tuned multilingual language model. Delivers * strong instruction adherence, multi-turn reasoning, and high-quality * content creation across 8+ core languages while maintaining a compact * on-device memory profile. */ - LLAMA3_2_3B: { - DEFAULT: LLAMA3_2_3B_SPINQUANT, + LLAMA3_2_3B: variants({ XNNPACK_SPINQUANT: LLAMA3_2_3B_SPINQUANT, XNNPACK_BF16: LLAMA3_2_3B_BF16, - }, + }), /** * Hugging Face SmolLM2 135M ultra-compact language model. Engineered for * micro-memory footprints, instant token generation, text classification, * and background processing on low-power devices. */ - SMOLLM2_135M: { - DEFAULT: SMOLLM2_135M_8DA4W, + SMOLLM2_135M: variants({ XNNPACK_8DA4W: SMOLLM2_135M_8DA4W, XNNPACK_BF16: SMOLLM2_135M_BF16, - }, + }), /** * Hugging Face SmolLM2 360M compact instruction-tuned model. Provides a * practical balance between fast mobile generation speed and conversational * coherence, ideal for lightweight on-device assistants, text * simplification, and structured data extraction. */ - SMOLLM2_360M: { - DEFAULT: SMOLLM2_360M_8DA4W, + SMOLLM2_360M: variants({ XNNPACK_8DA4W: SMOLLM2_360M_8DA4W, XNNPACK_BF16: SMOLLM2_360M_BF16, - }, + }), /** * Hugging Face SmolLM2 1.7B language model trained on curated educational, * synthetic, and web data. Delivers competitive reasoning, creative text * generation, and general knowledge Q&A performance approaching larger * 2B-3B models while maintaining fast on-device inference. */ - SMOLLM2_1_7B: { - DEFAULT: SMOLLM2_1_7B_8DA4W, + SMOLLM2_1_7B: variants({ XNNPACK_8DA4W: SMOLLM2_1_7B_8DA4W, XNNPACK_BF16: SMOLLM2_1_7B_BF16, - }, + }), /** * Hammer 2.1 0.5B specialized function-calling model. Fine-tuned * specifically for agentic tool use, structured JSON extraction, and * single/multi-tool invocation with ultra-low latency for real-time mobile * tool calling flows. */ - HAMMER2_1_0_5B: { - DEFAULT: HAMMER2_1_0_5B_XNNPACK_8DA4W, + HAMMER2_1_0_5B: variants({ XNNPACK_8DA4W: HAMMER2_1_0_5B_XNNPACK_8DA4W, XNNPACK_BF16: HAMMER2_1_0_5B_XNNPACK_BF16, - }, + }), /** * Hammer 2.1 1.5B function-calling language model. Optimized for multi-tool * agentic workflows, API parameter schema validation, and structured JSON * output generation on edge devices. */ - HAMMER2_1_1_5B: { - DEFAULT: HAMMER2_1_1_5B_XNNPACK_8DA4W, + HAMMER2_1_1_5B: variants({ XNNPACK_8DA4W: HAMMER2_1_1_5B_XNNPACK_8DA4W, XNNPACK_BF16: HAMMER2_1_1_5B_XNNPACK_BF16, - }, + }), /** * Hammer 2.1 3B high-capacity function-calling model. Provides top-tier * tool selection precision, multi-turn tool calling, error recovery, and * strict compliance with complex TypeScript/JSON schema specifications in * autonomous mobile agent pipelines. */ - HAMMER2_1_3B: { - DEFAULT: HAMMER2_1_3B_XNNPACK_8DA4W, + HAMMER2_1_3B: variants({ XNNPACK_8DA4W: HAMMER2_1_3B_XNNPACK_8DA4W, XNNPACK_BF16: HAMMER2_1_3B_XNNPACK_BF16, - }, + }), /** * Microsoft Phi-4 Mini 3.8B high-density reasoning model. Trained on * synthetic textbook-grade datasets for state-of-the-art on-device STEM * problem solving, complex mathematical reasoning, multi-step code * synthesis, and structured analytical tasks. */ - PHI4_MINI: { - DEFAULT: PHI4_MINI_XNNPACK_8DA4W, + PHI4_MINI: variants({ XNNPACK_8DA4W: PHI4_MINI_XNNPACK_8DA4W, XNNPACK_BF16: PHI4_MINI_XNNPACK_BF16, - }, + }), /** * Alibaba Qwen 2.5 0.5B ultra-lightweight multilingual model. Trained on * 18T tokens supporting 29+ languages; optimized for near-instant response * times, basic instruction following, multilingual translation, and * lightweight conversational assistants on mobile devices. */ - QWEN2_5_0_5B: { - DEFAULT: QWEN2_5_0_5B_XNNPACK_8DA4W, + QWEN2_5_0_5B: variants({ XNNPACK_8DA4W: QWEN2_5_0_5B_XNNPACK_8DA4W, XNNPACK_BF16: QWEN2_5_0_5B_XNNPACK_BF16, - }, + }), /** * Alibaba Qwen 2.5 1.5B multilingual instruction model. Combines broad * multilingual comprehension across 29+ languages with strong coding and * math capabilities, well suited for interactive chat, summarization, and * cross-lingual translation. */ - QWEN2_5_1_5B: { - DEFAULT: QWEN2_5_1_5B_XNNPACK_8DA4W, + QWEN2_5_1_5B: variants({ XNNPACK_8DA4W: QWEN2_5_1_5B_XNNPACK_8DA4W, XNNPACK_BF16: QWEN2_5_1_5B_XNNPACK_BF16, - }, + }), /** * Alibaba Qwen 2.5 3B high-capability multilingual model. Delivers strong * reasoning, coding, mathematics, and multilingual fluency across 29+ * languages for in-depth text generation and complex multi-turn dialogue. */ - QWEN2_5_3B: { - DEFAULT: QWEN2_5_3B_XNNPACK_8DA4W, + QWEN2_5_3B: variants({ XNNPACK_8DA4W: QWEN2_5_3B_XNNPACK_8DA4W, XNNPACK_BF16: QWEN2_5_3B_XNNPACK_BF16, - }, + }), /** * Alibaba Qwen 3 0.6B next-generation compact language model. Features * updated architectural optimizations for reduced latency, enhanced * multilingual token representation, and efficient conversational * turn-taking on mobile devices. */ - QWEN3_0_6B: { - DEFAULT: QWEN3_0_6B_XNNPACK_8DA4W, + QWEN3_0_6B: variants({ XNNPACK_8DA4W: QWEN3_0_6B_XNNPACK_8DA4W, XNNPACK_BF16: QWEN3_0_6B_XNNPACK_BF16, - }, + }), /** * Alibaba Qwen 3 1.7B next-generation multilingual language model. Balances * high reasoning capability, general knowledge retrieval, coding * proficiency, and conversational fluidity across multiple languages. */ - QWEN3_1_7B: { - DEFAULT: QWEN3_1_7B_XNNPACK_8DA4W, + QWEN3_1_7B: variants({ XNNPACK_8DA4W: QWEN3_1_7B_XNNPACK_8DA4W, XNNPACK_BF16: QWEN3_1_7B_XNNPACK_BF16, - }, + }), /** * Alibaba Qwen 3 4B high-capacity generative model. Delivers advanced * multi-step reasoning, comprehensive world knowledge, complex coding * capabilities, and top-tier multilingual performance for demanding * on-device AI applications. */ - QWEN3_4B: { - DEFAULT: QWEN3_4B_XNNPACK_8DA4W, + QWEN3_4B: variants({ XNNPACK_8DA4W: QWEN3_4B_XNNPACK_8DA4W, XNNPACK_BF16: QWEN3_4B_XNNPACK_BF16, - }, + }), /** * Google Gemma 4 E2B generative language model. Built on Google's Gemini * research and architecture innovations, offering high-fidelity instruction * following, creative text generation, and reasoning efficiency optimized * for mobile deployment. */ - GEMMA4_E2B: { - DEFAULT: GEMMA4_E2B_XNNPACK_8DA4W, + GEMMA4_E2B: variants({ XNNPACK_8DA4W: GEMMA4_E2B_XNNPACK_8DA4W, MLX_INT4: GEMMA4_E2B_MLX_INT4, - }, + }), }, /** @@ -2178,71 +2097,63 @@ export const models = { * vector space. Optimized for fast, general-purpose semantic search, * sentence similarity, and clustering. */ - ALL_MINILM_L6_V2: { - DEFAULT: ALL_MINILM_L6_V2_EMBEDDINGS, + ALL_MINILM_L6_V2: variants({ XNNPACK_FP32: ALL_MINILM_L6_V2_EMBEDDINGS, - }, + }), /** * High-quality 768-dimensional sentence transformer model based on MPNet. * Provides higher quality semantic embeddings compared to MiniLM. */ - ALL_MPNET_BASE_V2: { - DEFAULT: ALL_MPNET_BASE_V2_EMBEDDINGS, + ALL_MPNET_BASE_V2: variants({ XNNPACK_FP32: ALL_MPNET_BASE_V2_EMBEDDINGS, - }, + }), /** * 384-dimensional sentence transformer fine-tuned specifically for semantic * QA matching using cosine similarity. */ - MULTI_QA_MINILM_L6_COS_V1: { - DEFAULT: MULTI_QA_MINILM_L6_COS_V1_EMBEDDINGS, + MULTI_QA_MINILM_L6_COS_V1: variants({ XNNPACK_FP32: MULTI_QA_MINILM_L6_COS_V1_EMBEDDINGS, - }, + }), /** * 768-dimensional sentence transformer fine-tuned specifically for * question-answering matching using dot product distance. */ - MULTI_QA_MPNET_BASE_DOT_V1: { - DEFAULT: MULTI_QA_MPNET_BASE_DOT_V1_EMBEDDINGS, + MULTI_QA_MPNET_BASE_DOT_V1: variants({ XNNPACK_FP32: MULTI_QA_MPNET_BASE_DOT_V1_EMBEDDINGS, - }, + }), /** * 384-dimensional sentence transformer supporting 50+ languages for * cross-lingual semantic similarity. */ - PARAPHRASE_MULTILINGUAL_MINILM_L12_V2: { - DEFAULT: PARAPHRASE_MULTILINGUAL_MINILM_L12_V2_EMBEDDINGS, + PARAPHRASE_MULTILINGUAL_MINILM_L12_V2: variants({ XNNPACK_8DA4W: PARAPHRASE_MULTILINGUAL_MINILM_L12_V2_EMBEDDINGS, - }, + }), /** * Multilingual sentence transformer supporting 50+ languages, based on * distilled Universal Sentence Encoder (512-dim output). */ - DISTILUSE_BASE_MULTILINGUAL_CASED_V2: { - DEFAULT: DISTILUSE_BASE_MULTILINGUAL_CASED_V2_EMBEDDINGS, + DISTILUSE_BASE_MULTILINGUAL_CASED_V2: variants({ XNNPACK_8DA4W: DISTILUSE_BASE_MULTILINGUAL_CASED_V2_EMBEDDINGS, MLX_INT8: DISTILUSE_BASE_MULTILINGUAL_CASED_V2_MLX_INT8, - }, + }), /** * CLIP text encoder (ViT-B/32) mapping text queries into a 512-dimensional * joint text-image embedding space. Used in combination with * `imageEmbeddings.CLIP_VIT_BASE_PATCH32` for zero-shot text-to-image * search. */ - CLIP_VIT_BASE_PATCH32_TEXT: { - DEFAULT: CLIP_VIT_BASE_PATCH32_TEXT_EMBEDDINGS, + CLIP_VIT_BASE_PATCH32_TEXT: variants({ XNNPACK_FP32: CLIP_VIT_BASE_PATCH32_TEXT_EMBEDDINGS, - }, + }), /** * Liquid AI LFM 2.5 350M parameter embedding model for asymmetric search * and retrieval tasks. Prompts queries with `query: ` (the default) and * passages with `document: ` via {@link TextEmbedder.embed}. */ - LFM2_5_EMBEDDING_350M: { - DEFAULT: LFM2_5_EMBEDDING_350M_EMBEDDINGS, + LFM2_5_EMBEDDING_350M: variants({ XNNPACK_8DA4W: LFM2_5_EMBEDDING_350M_EMBEDDINGS, MLX_INT4: LFM2_5_EMBEDDING_350M_MLX_INT4, - }, + }), }, /** @@ -2255,20 +2166,18 @@ export const models = { * OpenAI-style detector covering 8 common PII types (name, email, phone, * address, and similar). Compact label space, best for general redaction. */ - OPENAI: { - DEFAULT: PRIVACY_FILTER_OPENAI_XNNPACK_8DA4W, + OPENAI: variants({ XNNPACK_8DA4W: PRIVACY_FILTER_OPENAI_XNNPACK_8DA4W, MLX_INT4: PRIVACY_FILTER_OPENAI_MLX_INT4, - }, + }), /** * Nemotron-based detector covering 55 fine-grained PII types. Larger label * space for stricter compliance-oriented redaction. */ - NEMOTRON: { - DEFAULT: PRIVACY_FILTER_NEMOTRON_XNNPACK_8DA4W, + NEMOTRON: variants({ XNNPACK_8DA4W: PRIVACY_FILTER_NEMOTRON_XNNPACK_8DA4W, MLX_INT8: PRIVACY_FILTER_NEMOTRON_MLX_INT8, - }, + }), }, /** @@ -2280,12 +2189,11 @@ export const models = { * shared text-image space. Used for zero-shot visual classification and * cross-modal image search. */ - CLIP_VIT_BASE_PATCH32: { - DEFAULT: CLIP_VIT_BASE_PATCH32_IMAGE_XNNPACK_FP32, + CLIP_VIT_BASE_PATCH32: variants({ XNNPACK_FP32: CLIP_VIT_BASE_PATCH32_IMAGE_XNNPACK_FP32, COREML_FP16: CLIP_VIT_BASE_PATCH32_IMAGE_COREML_FP16, MLX_INT8: CLIP_VIT_BASE_PATCH32_IMAGE_MLX_INT8, - }, + }), }, /** @@ -2297,11 +2205,10 @@ export const models = { * generation model based on DreamShaper. Generates high-quality images from * text prompts in real time. */ - SDXS_512_DREAMSHAPER: { - DEFAULT: SDXS_512_DREAMSHAPER_XNNPACK_FP32, + SDXS_512_DREAMSHAPER: variants({ XNNPACK_FP32: SDXS_512_DREAMSHAPER_XNNPACK_FP32, COREML_FP16: SDXS_512_DREAMSHAPER_COREML_FP16, - }, + }), }, /** @@ -2314,11 +2221,10 @@ export const models = { * natural, highly expressive speech synthesis with configurable speaker * voice presets (see {@link SUPERTONIC_DEFAULT_VOICE_NAMES}). */ - SUPERTONIC: { - DEFAULT: SUPERTONIC_3_XNNPACK_FP32, + SUPERTONIC: variants({ XNNPACK_FP32: SUPERTONIC_3_XNNPACK_FP32, MLX_FP32: SUPERTONIC_3_MLX_FP32, - }, + }), /** * Kokoro — a lightweight phoneme-driven Text-to-Speech model. Each language @@ -2326,42 +2232,33 @@ export const models = { * the voices available for that language, nested per backend. */ KOKORO: { - EN_US: { - DEFAULT: KOKORO_EN_US_XNNPACK_FP32, + EN_US: variants({ XNNPACK_FP32: KOKORO_EN_US_XNNPACK_FP32, - }, - EN_GB: { - DEFAULT: KOKORO_EN_GB_XNNPACK_FP32, + }), + EN_GB: variants({ XNNPACK_FP32: KOKORO_EN_GB_XNNPACK_FP32, - }, - ES: { - DEFAULT: KOKORO_ES_XNNPACK_FP32, + }), + ES: variants({ XNNPACK_FP32: KOKORO_ES_XNNPACK_FP32, - }, - FR: { - DEFAULT: KOKORO_FR_XNNPACK_FP32, + }), + FR: variants({ XNNPACK_FP32: KOKORO_FR_XNNPACK_FP32, - }, - IT: { - DEFAULT: KOKORO_IT_XNNPACK_FP32, + }), + IT: variants({ XNNPACK_FP32: KOKORO_IT_XNNPACK_FP32, - }, - PT: { - DEFAULT: KOKORO_PT_XNNPACK_FP32, + }), + PT: variants({ XNNPACK_FP32: KOKORO_PT_XNNPACK_FP32, - }, - HI: { - DEFAULT: KOKORO_HI_XNNPACK_FP32, + }), + HI: variants({ XNNPACK_FP32: KOKORO_HI_XNNPACK_FP32, - }, - PL: { - DEFAULT: KOKORO_PL_XNNPACK_FP32, + }), + PL: variants({ XNNPACK_FP32: KOKORO_PL_XNNPACK_FP32, - }, - DE: { - DEFAULT: KOKORO_DE_XNNPACK_FP32, + }), + DE: variants({ XNNPACK_FP32: KOKORO_DE_XNNPACK_FP32, - }, + }), }, }, @@ -2380,12 +2277,11 @@ export const models = { /** * PP-OCRv6 Small multilingual OCR model. */ - PPOCRV6_SMALL: { - DEFAULT: PPOCRV6_SMALL_XNNPACK_INT8, + PPOCRV6_SMALL: variants({ XNNPACK: PPOCRV6_SMALL_XNNPACK_INT8, COREML: PPOCRV6_SMALL_COREML_INT8, VULKAN: PPOCRV6_SMALL_VULKAN_FP16, - }, + }), }, }, }; From 6ea2b97f150ea6b578d70e5a4af42a517bfff5fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20S=C5=82uszniak?= Date: Thu, 27 Aug 2026 17:22:10 +0200 Subject: [PATCH 02/24] feat(models): prefer MLX on iOS and Vulkan on Android An accelerated export only gets published once it beats the CPU one, so a published variant is itself the signal to prefer it. MLX now sits above XNNPACK on iOS and Vulkan above XNNPACK on Android, rather than both being an explicit opt-in. Core ML still leads on iOS, but only to give the pair a deterministic order. The 8 groups publishing both Core ML and MLX -- 6 Whisper sizes, RF-DETR keypoint and the CLIP vision encoder -- now pin their winner at the call site with the reason, and a test fails until a new such group does the same. On Android this moves both LFM2.5-VL sizes and PP-OCRv6 to Vulkan; on iOS it moves 4 LLMs, 2 text embedders, both privacy filters and Supertonic TTS to MLX. --- .../01-fundamentals/01-getting-started.md | 11 +- .../__tests__/api/modelVariants.test.ts | 102 ++++++++++++-- .../src/modelVariants.ts | 25 ++-- .../react-native-executorch/src/models.ts | 131 +++++++++++------- 4 files changed, 198 insertions(+), 71 deletions(-) diff --git a/docs/docs/01-fundamentals/01-getting-started.md b/docs/docs/01-fundamentals/01-getting-started.md index 61e449936b..0f814c5d7c 100644 --- a/docs/docs/01-fundamentals/01-getting-started.md +++ b/docs/docs/01-fundamentals/01-getting-started.md @@ -247,13 +247,14 @@ models.classification.EFFICIENTNET_V2_S.COREML_FP16; // always this export models.classification.EFFICIENTNET_V2_S.XNNPACK_INT8; ``` -`DEFAULT` is not a fixed file. It is resolved when the library loads, to the fastest export the device can actually run: +`DEFAULT` is not a fixed file. It is resolved when the library loads, to the fastest export the device can actually run. A model is only exported to an accelerated backend once it has been shown to run better there, so a published accelerated variant is preferred and XNNPACK is the fallback: -- **iOS device** — the Core ML export where one exists, so the model reaches the Neural Engine. -- **Android, and the iOS simulator** — the XNNPACK export. The simulator has no Neural Engine and cannot run Core ML models at all. -- **Either, narrowed by your config** — only backends your app downloaded are considered. Trimming `coreml` out of an iOS build moves every `DEFAULT` back to XNNPACK rather than failing to load. +- **iOS device** — Core ML where one exists, otherwise MLX, otherwise XNNPACK. +- **Android** — Vulkan where one exists, otherwise XNNPACK. +- **iOS simulator** — XNNPACK. The simulator has no Neural Engine, cannot run Core ML models at all, and MLX ships a device slice only. +- **All of them, narrowed by your config** — only backends your app downloaded are considered. Trimming `coreml` out of an iOS build moves those `DEFAULT`s to the next best export rather than failing to load. -MLX and Vulkan are never picked automatically: they win on some models and lose on others, so reach for `MLX_*` / `VULKAN_*` explicitly when you have benchmarked your case. Naming any variant directly always overrides the resolution. +A handful of models publish both a Core ML and an MLX export. There the two are close enough that the winner is a per-model benchmark result, so the registry pins it explicitly rather than letting the order above decide. Naming any variant directly always overrides the resolution. ### Platform notes diff --git a/packages/react-native-executorch/__tests__/api/modelVariants.test.ts b/packages/react-native-executorch/__tests__/api/modelVariants.test.ts index e600396585..e33e54ab5a 100644 --- a/packages/react-native-executorch/__tests__/api/modelVariants.test.ts +++ b/packages/react-native-executorch/__tests__/api/modelVariants.test.ts @@ -6,6 +6,9 @@ * below therefore reloads the registry behind a `jest.resetModules()` rather * than reading the copy the test file imported. */ +import { readFileSync } from 'fs'; +import { join } from 'path'; + import { Platform } from 'react-native'; import { fakeJsi } from '../support/fakeJsi'; @@ -87,6 +90,36 @@ function registryFor(options: { return require('../../src/models').models as Node; } +/** + * Every `variants(...)` call in the registry source, with the variant keys it + * lists and whether it pins one for iOS. + * + * Prettier gives the two call shapes distinct first lines — `variants({` when + * the group takes no pins, `variants(` when it does — which is what this reads. + * @returns One entry per call, in source order. + */ +function variantsCalls(): { name: string; keys: string[]; pinsIos: boolean }[] { + const source = readFileSync(join(__dirname, '../../src/models.ts'), 'utf8').split('\n'); + const calls: { name: string; keys: string[]; pinsIos: boolean }[] = []; + + for (let line = 0; line < source.length; line++) { + const opened = source[line]!.match(/^(\s*)([A-Z][A-Z0-9_]*): variants\((\{?)$/); + if (!opened) continue; + const [, indent, name, inlineBrace] = opened; + + const closer = inlineBrace ? `${indent}}),` : `${indent}),`; + let end = line + 1; + while (end < source.length && source[end] !== closer) end++; + + const body = source.slice(line + 1, end); + const keys = body.flatMap((entry) => entry.match(/^\s*([A-Z][A-Z0-9_]*):/)?.slice(1) ?? []); + calls.push({ name: name!, keys, pinsIos: body.some((entry) => /\bios:/.test(entry)) }); + line = end; + } + + return calls; +} + /** Every group of the reloaded registry, paired with the key it defaulted to. */ const defaultsOf = (registry: Node) => variantGroups(registry).map(({ label, group }) => ({ @@ -123,26 +156,77 @@ describe('DEFAULT variant resolution', () => { expect(offenders).toEqual([]); }); - it('defaults to XNNPACK on Android wherever an XNNPACK export exists', () => { + it('defaults to XNNPACK on Android where no Vulkan export exists', () => { const offenders = defaultsOf(registryFor({ os: 'android' })) .filter(({ offers }) => offers.some((key) => key.startsWith('XNNPACK'))) + .filter(({ offers }) => !offers.some((key) => key.startsWith('VULKAN'))) .filter(({ key }) => key !== undefined && !key.startsWith('XNNPACK')) .map(({ label, key }) => `${label}: ${key}`); expect(offenders).toEqual([]); }); - it('leaves Vulkan and MLX as an explicit opt-in on both platforms', () => { - for (const os of ['ios', 'android'] as const) { - const offenders = defaultsOf(registryFor({ os })) - .filter(({ key }) => key?.startsWith('VULKAN') || key?.startsWith('MLX')) - .map(({ label, key }) => `${os} ${label}: ${key}`); + it('prefers MLX over XNNPACK on iOS where no Core ML export exists', () => { + const offenders = defaultsOf(registryFor({ os: 'ios' })) + .filter(({ offers }) => offers.some((key) => key.startsWith('MLX'))) + .filter(({ offers }) => !offers.some((key) => key.startsWith('COREML'))) + .filter(({ key }) => key !== undefined && !key.startsWith('MLX')) + .map(({ label, key }) => `${label}: ${key}`); - expect(offenders).toEqual([]); - } + expect(offenders).toEqual([]); + }); + + it('prefers Vulkan over XNNPACK on Android wherever a Vulkan export exists', () => { + const offenders = defaultsOf(registryFor({ os: 'android' })) + .filter(({ offers }) => offers.some((key) => key.startsWith('VULKAN'))) + .filter(({ key }) => key !== undefined && !key.startsWith('VULKAN')) + .map(({ label, key }) => `${label}: ${key}`); + + expect(offenders).toEqual([]); + }); + + it('never picks Vulkan on iOS or an iOS-only backend on Android', () => { + expect( + defaultsOf(registryFor({ os: 'ios' })) + .filter(({ key }) => key?.startsWith('VULKAN')) + .map(({ label, key }) => `${label}: ${key}`) + ).toEqual([]); + + expect( + defaultsOf(registryFor({ os: 'android' })) + .filter(({ key }) => key?.startsWith('COREML') || key?.startsWith('MLX')) + .map(({ label, key }) => `${label}: ${key}`) + ).toEqual([]); + }); + + it('reads every variant group out of the registry source', () => { + // The pin case below is a source-level check, so it passes for free if the + // scanner stops matching the shape Prettier writes. + const calls = variantsCalls(); + expect(calls.length).toBeGreaterThan(100); + expect(calls.filter(({ pinsIos }) => pinsIos).length).toBeGreaterThan(0); + expect(calls.filter(({ keys }) => keys.length === 0)).toEqual([]); + }); + + it('every group offering both Core ML and MLX pins one', () => { + // Core ML sits above MLX in the backend order only to make the resolution + // deterministic, and that ordering is not a benchmark result. Where a model + // publishes both, the winner has to be written down at the call site so the + // choice is reviewable rather than an accident of the enum order. + // + // A pin to the backend the order would have picked anyway is invisible at + // runtime, so this reads the registry source rather than the resolved + // registry. + const offenders = variantsCalls() + .filter(({ keys }) => keys.some((key) => key.startsWith('COREML'))) + .filter(({ keys }) => keys.some((key) => key.startsWith('MLX'))) + .filter(({ pinsIos }) => !pinsIos) + .map(({ name, keys }) => `${name}: ${keys.join(', ')}`); + + expect(offenders).toEqual([]); }); - it('falls back to XNNPACK on the iOS simulator, which cannot run Core ML', () => { + it('falls back to XNNPACK on the iOS simulator, which runs neither Core ML nor MLX', () => { const offenders = defaultsOf(registryFor({ os: 'ios', isEmulator: true })) .filter(({ path }) => /\/(coreml|mlx)\//.test(path)) .map(({ label, path }) => `${label}: ${path}`); diff --git a/packages/react-native-executorch/src/modelVariants.ts b/packages/react-native-executorch/src/modelVariants.ts index 7f82d3fddb..5775f1f729 100644 --- a/packages/react-native-executorch/src/modelVariants.ts +++ b/packages/react-native-executorch/src/modelVariants.ts @@ -18,11 +18,11 @@ * otherwise"). * * A model whose best variant does not follow from that ordering can pin one - * per platform — see the second argument of {@link variants}. - * - * Backends the resolver never picks on its own — MLX on iOS, Vulkan on - * Android — stay reachable through their explicit keys. They win on some - * models and lose on others, so they are an opt-in rather than a default. + * per platform — see the second argument of {@link variants}. A model that + * publishes both a Core ML and an MLX export has to: the two are close enough + * that the winner is a per-model benchmark result, not something an ordering + * can state, so `every group offering both Core ML and MLX pins one` in + * `__tests__/api/modelVariants.test.ts` fails until the choice is written down. * @module ModelVariants * @internal */ @@ -40,13 +40,18 @@ type TargetPlatform = 'ios' | 'android'; /** * Backends to try, best first, per platform. * - * Core ML leads on iOS: it reaches the Neural Engine, which beats XNNPACK's - * CPU kernels on every model family that ships both. XNNPACK leads on Android - * and backs iOS up, because it is the one backend every model exports to. + * The accelerated backends lead and XNNPACK trails on both platforms: a model + * is only exported to Core ML, MLX or Vulkan once it has been shown to run + * better there, so a published accelerated variant is itself the signal that it + * should be preferred. XNNPACK is the fallback because it is the one backend + * every model exports to. + * + * Core ML sits above MLX on iOS only to give the pair a deterministic order; + * every model that publishes both has to pin its winner explicitly. */ const BACKEND_ORDER: Record = { - ios: ['COREML', 'XNNPACK'], - android: ['XNNPACK'], + ios: ['COREML', 'MLX', 'XNNPACK'], + android: ['VULKAN', 'XNNPACK'], }; /** Variant keys pinned per platform, overriding {@link BACKEND_ORDER}. */ diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts index b2a52d74d2..d907d920a8 100644 --- a/packages/react-native-executorch/src/models.ts +++ b/packages/react-native-executorch/src/models.ts @@ -1343,9 +1343,10 @@ const QWEN3_4B_XNNPACK_BF16: LLMModel = { * * Models published for more than one backend expose their exports as named * variants (`XNNPACK_INT8`, `COREML_FP16`, ...) plus a `DEFAULT` alias. The - * alias is chosen for the device the app runs on: Core ML on iOS hardware, - * XNNPACK on Android and on the iOS simulator, always narrowed to the backends - * the app actually linked in. Reach for a named variant to override that. + * alias is chosen for the device the app runs on: Core ML then MLX on iOS + * hardware, Vulkan on Android, XNNPACK as the fallback everywhere and the only + * option on the iOS simulator — always narrowed to the backends the app + * actually linked in. Reach for a named variant to override that. * @category Models */ export const models = { @@ -1646,11 +1647,16 @@ export const models = { * RF-DETR (Roboflow Detection Transformer) pose keypoint detector * predicting 17 COCO body keypoints (see {@link COCO_LANDMARKS}). */ - RFDETR_KEYPOINT: variants({ - XNNPACK_FP32: RFDETR_KEYPOINT_XNNPACK_FP32, - COREML_FP32: RFDETR_KEYPOINT_COREML_FP32, - MLX_FP32: RFDETR_KEYPOINT_MLX_FP32, - }), + RFDETR_KEYPOINT: variants( + { + XNNPACK_FP32: RFDETR_KEYPOINT_XNNPACK_FP32, + COREML_FP32: RFDETR_KEYPOINT_COREML_FP32, + MLX_FP32: RFDETR_KEYPOINT_MLX_FP32, + }, + // Core ML over MLX: the MLX delegate runs on the GPU, and #1318 measured + // it losing to the Neural Engine across the vision models that ship both. + { ios: 'COREML_FP32' } + ), }, /** @@ -1797,66 +1803,93 @@ export const models = { * Voice Activity Detection. Includes multilingual and English-only (`EN`) * variants across model sizes (`TINY`, `BASE`, `SMALL`). */ + // Every size defaults to Core ML over MLX on iOS: the MLX encoder has to + // hold the whole mel spectrogram, and the bf16 builds are jetsam-killed on + // an iPhone 16. Reach for MLX_INT8 explicitly if you want the GPU path. WHISPER: { /** * Multilingual Whisper Tiny model. Supporting 99+ languages. High speed * speech recognition. */ - TINY: variants({ - XNNPACK_FP32: WHISPER_TINY_XNNPACK_FP32, - COREML_FP16: WHISPER_TINY_COREML_FP16, - MLX_BF16: WHISPER_TINY_MLX_BF16, - MLX_INT8: WHISPER_TINY_MLX_INT8, - }), + TINY: variants( + { + XNNPACK_FP32: WHISPER_TINY_XNNPACK_FP32, + COREML_FP16: WHISPER_TINY_COREML_FP16, + MLX_BF16: WHISPER_TINY_MLX_BF16, + MLX_INT8: WHISPER_TINY_MLX_INT8, + }, + // Core ML over MLX, see the note on WHISPER. + { ios: 'COREML_FP16' } + ), /** * Multilingual Whisper Base model. Higher accuracy across supported * languages. */ - BASE: variants({ - XNNPACK_FP32: WHISPER_BASE_XNNPACK_FP32, - COREML_FP16: WHISPER_BASE_COREML_FP16, - MLX_BF16: WHISPER_BASE_MLX_BF16, - MLX_INT8: WHISPER_BASE_MLX_INT8, - }), + BASE: variants( + { + XNNPACK_FP32: WHISPER_BASE_XNNPACK_FP32, + COREML_FP16: WHISPER_BASE_COREML_FP16, + MLX_BF16: WHISPER_BASE_MLX_BF16, + MLX_INT8: WHISPER_BASE_MLX_INT8, + }, + // Core ML over MLX, see the note on WHISPER. + { ios: 'COREML_FP16' } + ), /** * Multilingual Whisper Small model. Best accuracy for complex * multi-language audio. */ - SMALL: variants({ - XNNPACK_FP32: WHISPER_SMALL_XNNPACK_FP32, - COREML_FP16: WHISPER_SMALL_COREML_FP16, - MLX_INT8: WHISPER_SMALL_MLX_INT8, - }), + SMALL: variants( + { + XNNPACK_FP32: WHISPER_SMALL_XNNPACK_FP32, + COREML_FP16: WHISPER_SMALL_COREML_FP16, + MLX_INT8: WHISPER_SMALL_MLX_INT8, + }, + // Core ML over MLX, see the note on WHISPER. + { ios: 'COREML_FP16' } + ), /** English-only optimized Whisper models (`TINY`, `BASE`, `SMALL`). */ EN: { /** * English-only Whisper Tiny model. Fast and compact for English STT. */ - TINY: variants({ - XNNPACK_FP32: WHISPER_TINY_EN_XNNPACK_FP32, - COREML_FP16: WHISPER_TINY_EN_COREML_FP16, - MLX_BF16: WHISPER_TINY_EN_MLX_BF16, - MLX_INT8: WHISPER_TINY_EN_MLX_INT8, - }), + TINY: variants( + { + XNNPACK_FP32: WHISPER_TINY_EN_XNNPACK_FP32, + COREML_FP16: WHISPER_TINY_EN_COREML_FP16, + MLX_BF16: WHISPER_TINY_EN_MLX_BF16, + MLX_INT8: WHISPER_TINY_EN_MLX_INT8, + }, + // Core ML over MLX, see the note on WHISPER. + { ios: 'COREML_FP16' } + ), /** * English-only Whisper Base model. High accuracy English speech * recognition. */ - BASE: variants({ - XNNPACK_FP32: WHISPER_BASE_EN_XNNPACK_FP32, - COREML_FP16: WHISPER_BASE_EN_COREML_FP16, - MLX_BF16: WHISPER_BASE_EN_MLX_BF16, - MLX_INT8: WHISPER_BASE_EN_MLX_INT8, - }), + BASE: variants( + { + XNNPACK_FP32: WHISPER_BASE_EN_XNNPACK_FP32, + COREML_FP16: WHISPER_BASE_EN_COREML_FP16, + MLX_BF16: WHISPER_BASE_EN_MLX_BF16, + MLX_INT8: WHISPER_BASE_EN_MLX_INT8, + }, + // Core ML over MLX, see the note on WHISPER. + { ios: 'COREML_FP16' } + ), /** * English-only Whisper Small model. Superior accuracy for English * transcription. */ - SMALL: variants({ - XNNPACK_FP32: WHISPER_SMALL_EN_XNNPACK_FP32, - COREML_FP16: WHISPER_SMALL_EN_COREML_FP16, - MLX_INT8: WHISPER_SMALL_EN_MLX_INT8, - }), + SMALL: variants( + { + XNNPACK_FP32: WHISPER_SMALL_EN_XNNPACK_FP32, + COREML_FP16: WHISPER_SMALL_EN_COREML_FP16, + MLX_INT8: WHISPER_SMALL_EN_MLX_INT8, + }, + // Core ML over MLX, see the note on WHISPER. + { ios: 'COREML_FP16' } + ), }, }, }, @@ -2189,11 +2222,15 @@ export const models = { * shared text-image space. Used for zero-shot visual classification and * cross-modal image search. */ - CLIP_VIT_BASE_PATCH32: variants({ - XNNPACK_FP32: CLIP_VIT_BASE_PATCH32_IMAGE_XNNPACK_FP32, - COREML_FP16: CLIP_VIT_BASE_PATCH32_IMAGE_COREML_FP16, - MLX_INT8: CLIP_VIT_BASE_PATCH32_IMAGE_MLX_INT8, - }), + CLIP_VIT_BASE_PATCH32: variants( + { + XNNPACK_FP32: CLIP_VIT_BASE_PATCH32_IMAGE_XNNPACK_FP32, + COREML_FP16: CLIP_VIT_BASE_PATCH32_IMAGE_COREML_FP16, + MLX_INT8: CLIP_VIT_BASE_PATCH32_IMAGE_MLX_INT8, + }, + // Core ML over MLX, as for every other vision encoder — see #1318. + { ios: 'COREML_FP16' } + ), }, /** From 923bd3861eddb202f8f3a1cbe43fdca30d51ccd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20S=C5=82uszniak?= Date: Thu, 27 Aug 2026 18:32:10 +0200 Subject: [PATCH 03/24] feat(models): add the fp16 Core ML build of RF-DETR keypoint RF-DETR keypoint was the registry's only fp32 Core ML model, and the Neural Engine is fp16-only, so it could never reach the ANE whatever compute unit it asked for. Measured on an iPhone 16, raw execute, 20 iterations: coreml fp16 122.7 ms 75 MB coreml fp32 159.9 ms 148 MB (published) mlx fp32 270.3 ms 145 MB A compute-unit sweep confirms the cause: restricting fp32 to CPU_AND_NE costs 2x (337.5 ms) and matches CPU_ONLY, so the ANE contributes nothing at fp32; the same restriction on fp16 costs 2 ms. ALL stays the best compute unit at both precisions, which is the ExecuTorch default the export scripts already rely on. The build is published alongside the fp32 one, same graph and same I/O ([1,3,576,576] in, [100,4]/[100]/[100,17,3] out), and iOS now pins to it. --- packages/react-native-executorch/src/models.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts index d907d920a8..cd4bead878 100644 --- a/packages/react-native-executorch/src/models.ts +++ b/packages/react-native-executorch/src/models.ts @@ -490,6 +490,10 @@ const RFDETR_KEYPOINT_XNNPACK_FP32: KeypointDetectorModel<'xyxy', CocoLandmark> modelPath: `${BASE_URL}-rfdetr-keypoint/${VERSION_TAG}/preview/xnnpack/rfdetr_keypoint_preview_xnnpack_fp32.pte`, modelOpts: RFDETR_KEYPOINT_OPTS, }; +const RFDETR_KEYPOINT_COREML_FP16: KeypointDetectorModel<'xyxy', CocoLandmark> = { + modelPath: `${BASE_URL}-rfdetr-keypoint/${NEXT_VERSION_TAG}/preview/coreml/rfdetr_keypoint_preview_coreml_fp16.pte`, + modelOpts: RFDETR_KEYPOINT_OPTS, +}; const RFDETR_KEYPOINT_COREML_FP32: KeypointDetectorModel<'xyxy', CocoLandmark> = { modelPath: `${BASE_URL}-rfdetr-keypoint/${VERSION_TAG}/preview/coreml/rfdetr_keypoint_preview_coreml_fp32.pte`, modelOpts: RFDETR_KEYPOINT_OPTS, @@ -1650,12 +1654,16 @@ export const models = { RFDETR_KEYPOINT: variants( { XNNPACK_FP32: RFDETR_KEYPOINT_XNNPACK_FP32, + COREML_FP16: RFDETR_KEYPOINT_COREML_FP16, COREML_FP32: RFDETR_KEYPOINT_COREML_FP32, MLX_FP32: RFDETR_KEYPOINT_MLX_FP32, }, - // Core ML over MLX: the MLX delegate runs on the GPU, and #1318 measured - // it losing to the Neural Engine across the vision models that ship both. - { ios: 'COREML_FP32' } + // Core ML fp16 over MLX, measured on an iPhone 16 at 20 iterations: + // 122.7 ms against 270.3 ms for MLX and 159.9 ms for the fp32 Core ML + // build. The Neural Engine is fp16-only, which is why fp32 never reached + // it — restricting fp32 to CPU_AND_NE costs 2x, while doing the same to + // fp16 costs nothing. + { ios: 'COREML_FP16' } ), }, From 8a236a09ae589af4abb364c85a4bc9886ebea57d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20S=C5=82uszniak?= Date: Fri, 28 Aug 2026 07:59:56 +0200 Subject: [PATCH 04/24] Revert "feat(models): add the fp16 Core ML build of RF-DETR keypoint" This reverts commit 923bd3861. The fp16 build does not work. Run on device against a real photo, it returns zero detections where both fp32 builds return the same person at 0.863 confidence with matching boxes and landmarks: coreml-fp32 1 detection, conf 0.863, box [280.5, 43.1, 496.8, 391.2] xnnpack-fp32 1 detection, identical to within 0.01 px coreml-fp16 0 detections The benchmark that justified the pin ran on a synthetic scene that detects nothing, so it measured speed on a model that produces no output. Reverting the pin until the fp16 export is fixed and checked against real input. --- packages/react-native-executorch/src/models.ts | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts index cd4bead878..d907d920a8 100644 --- a/packages/react-native-executorch/src/models.ts +++ b/packages/react-native-executorch/src/models.ts @@ -490,10 +490,6 @@ const RFDETR_KEYPOINT_XNNPACK_FP32: KeypointDetectorModel<'xyxy', CocoLandmark> modelPath: `${BASE_URL}-rfdetr-keypoint/${VERSION_TAG}/preview/xnnpack/rfdetr_keypoint_preview_xnnpack_fp32.pte`, modelOpts: RFDETR_KEYPOINT_OPTS, }; -const RFDETR_KEYPOINT_COREML_FP16: KeypointDetectorModel<'xyxy', CocoLandmark> = { - modelPath: `${BASE_URL}-rfdetr-keypoint/${NEXT_VERSION_TAG}/preview/coreml/rfdetr_keypoint_preview_coreml_fp16.pte`, - modelOpts: RFDETR_KEYPOINT_OPTS, -}; const RFDETR_KEYPOINT_COREML_FP32: KeypointDetectorModel<'xyxy', CocoLandmark> = { modelPath: `${BASE_URL}-rfdetr-keypoint/${VERSION_TAG}/preview/coreml/rfdetr_keypoint_preview_coreml_fp32.pte`, modelOpts: RFDETR_KEYPOINT_OPTS, @@ -1654,16 +1650,12 @@ export const models = { RFDETR_KEYPOINT: variants( { XNNPACK_FP32: RFDETR_KEYPOINT_XNNPACK_FP32, - COREML_FP16: RFDETR_KEYPOINT_COREML_FP16, COREML_FP32: RFDETR_KEYPOINT_COREML_FP32, MLX_FP32: RFDETR_KEYPOINT_MLX_FP32, }, - // Core ML fp16 over MLX, measured on an iPhone 16 at 20 iterations: - // 122.7 ms against 270.3 ms for MLX and 159.9 ms for the fp32 Core ML - // build. The Neural Engine is fp16-only, which is why fp32 never reached - // it — restricting fp32 to CPU_AND_NE costs 2x, while doing the same to - // fp16 costs nothing. - { ios: 'COREML_FP16' } + // Core ML over MLX: the MLX delegate runs on the GPU, and #1318 measured + // it losing to the Neural Engine across the vision models that ship both. + { ios: 'COREML_FP32' } ), }, From c7fee750848d776815d56265a2b1738428faab0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20S=C5=82uszniak?= Date: Fri, 28 Aug 2026 09:45:20 +0200 Subject: [PATCH 05/24] docs(models): record the real reason behind each iOS pin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All three rationales were written before the measurements that settled them, and two of them state a mechanism that turned out to be wrong. Whisper was pinned to Core ML on memory grounds alone, because at the time it was losing the pipeline benchmark at tiny and base. It no longer is: the republished builds run `decode` on the CPU rather than the Neural Engine (export-scripts MR !18) and are 2.5-3.1x faster end to end on real speech, so the pin now rests on speed, memory and accuracy rather than memory alone. RF-DETR keypoint credited its win to the Neural Engine. It cannot be: the ANE is fp16-only and this is the registry's only fp32 Core ML model, which a compute-unit sweep confirms — `cpu_and_ne` matches `cpu_only`, and `all` matches `cpu_and_gpu`. Records the fp16 attempt too, which is faster and returns zero detections. CLIP cited an issue number instead of its numbers. --- .../react-native-executorch/src/models.ts | 25 ++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts index d907d920a8..3c4ae5af70 100644 --- a/packages/react-native-executorch/src/models.ts +++ b/packages/react-native-executorch/src/models.ts @@ -1653,8 +1653,13 @@ export const models = { COREML_FP32: RFDETR_KEYPOINT_COREML_FP32, MLX_FP32: RFDETR_KEYPOINT_MLX_FP32, }, - // Core ML over MLX: the MLX delegate runs on the GPU, and #1318 measured - // it losing to the Neural Engine across the vision models that ship both. + // Core ML over MLX: 164.1 ms against 272.3 on an iPhone 16, at 378 MB + // against 1304 MB. Note this is not a Neural Engine win — the ANE is + // fp16-only, so this fp32 program never reaches it, and a compute-unit + // sweep confirms it (`cpu_and_ne` 337.5 ms matches `cpu_only` 352.9, + // while `all` 173.6 matches `cpu_and_gpu` 176.7). An fp16 build would + // reach the ANE and is 23% faster, but returns zero detections on a real + // photo: the score head does not survive fp16. { ios: 'COREML_FP32' } ), }, @@ -1803,9 +1808,16 @@ export const models = { * Voice Activity Detection. Includes multilingual and English-only (`EN`) * variants across model sizes (`TINY`, `BASE`, `SMALL`). */ - // Every size defaults to Core ML over MLX on iOS: the MLX encoder has to - // hold the whole mel spectrogram, and the bf16 builds are jetsam-killed on - // an iPhone 16. Reach for MLX_INT8 explicitly if you want the GPU path. + // Every size defaults to Core ML over MLX on iOS, which wins on all three + // axes. Speed: 2.5-3.1x faster end to end on 14.5s of real speech, once the + // Core ML builds stopped dispatching their single-token `decode` to the + // Neural Engine (export-scripts MR !18 — decode is dispatch-bound, so + // CPU_ONLY runs it at 6.7 ms/step against 20.0). Memory: base peaks at + // 532 MB against 1834 MB for MLX bf16, which is jetsam territory on an + // iPhone 16 — MLX bf16 at small does not load at all. Accuracy: on the same + // clip MLX bf16 misheard "brown fox" as "round box" and "compute unit" as + // "computer unit", where Core ML matched the reference exactly. + // Reach for MLX_INT8 explicitly if you want the GPU path. WHISPER: { /** * Multilingual Whisper Tiny model. Supporting 99+ languages. High speed @@ -2228,7 +2240,8 @@ export const models = { COREML_FP16: CLIP_VIT_BASE_PATCH32_IMAGE_COREML_FP16, MLX_INT8: CLIP_VIT_BASE_PATCH32_IMAGE_MLX_INT8, }, - // Core ML over MLX, as for every other vision encoder — see #1318. + // Core ML over MLX: 3.5 ms against 14.1 on an iPhone 16, the widest + // margin of any pair that ships both. { ios: 'COREML_FP16' } ), }, From 58b18a4a71bf2da22fa303033a981e33ed8b79b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20S=C5=82uszniak?= Date: Fri, 28 Aug 2026 10:14:33 +0200 Subject: [PATCH 06/24] docs(models): fp16 RF-DETR keypoint is blocked on the export env, not precision The earlier note said the score head does not survive fp16. That compared a local fp16 build against the published fp32 one, which confounds precision with build provenance. Built from the same trace, fp16 and fp32 agree to 0.0008 confidence on device while both score 3x below the published fp32, so the gap is the export environment. --- packages/react-native-executorch/src/models.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts index 3c4ae5af70..2f309634c3 100644 --- a/packages/react-native-executorch/src/models.ts +++ b/packages/react-native-executorch/src/models.ts @@ -1658,8 +1658,11 @@ export const models = { // fp16-only, so this fp32 program never reaches it, and a compute-unit // sweep confirms it (`cpu_and_ne` 337.5 ms matches `cpu_only` 352.9, // while `all` 173.6 matches `cpu_and_gpu` 176.7). An fp16 build would - // reach the ANE and is 23% faster, but returns zero detections on a real - // photo: the score head does not survive fp16. + // reach the ANE and is 23% faster; exporting one is open, and blocked on + // the export environment rather than on precision. fp16 and fp32 built + // from the same trace agree to 0.0008 confidence on device, but both + // score 3x below the published fp32, and that build's `rfdetr` leaves + // `keypoint_head.keypoint_proj.*` unconsumed when loading the checkpoint. { ios: 'COREML_FP32' } ), }, From 2130f9d2166e312e854a2cb0e0710691407cc38c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20S=C5=82uszniak?= Date: Fri, 28 Aug 2026 13:44:23 +0200 Subject: [PATCH 07/24] feat(models): default RF-DETR keypoint to the new fp16 Core ML build on iOS An fp16 Core ML build is now published for this model, and it is faster and smaller than the fp32 one it replaces as the iOS default: 144.0 ms against 165.1, a 263 MB peak against 389 MB, and half the download, matching fp32 to 0.0008 confidence and 0.23 px on an iPhone 16. fp32 stays available as COREML_FP32. The comment on the pin records why this build is shaped the way it is. fp16 is correct on device only with the CPU and the Neural Engine both excluded, and only at an iOS17 deployment target, and neither constraint is visible from a host-side check. Those live in export-scripts MR !18; the note here exists so nobody "simplifies" the pin without knowing what it is holding. --- .../react-native-executorch/src/models.ts | 28 ++++++++++++------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts index 2f309634c3..770ce4bc2d 100644 --- a/packages/react-native-executorch/src/models.ts +++ b/packages/react-native-executorch/src/models.ts @@ -490,6 +490,10 @@ const RFDETR_KEYPOINT_XNNPACK_FP32: KeypointDetectorModel<'xyxy', CocoLandmark> modelPath: `${BASE_URL}-rfdetr-keypoint/${VERSION_TAG}/preview/xnnpack/rfdetr_keypoint_preview_xnnpack_fp32.pte`, modelOpts: RFDETR_KEYPOINT_OPTS, }; +const RFDETR_KEYPOINT_COREML_FP16: KeypointDetectorModel<'xyxy', CocoLandmark> = { + modelPath: `${BASE_URL}-rfdetr-keypoint/${VERSION_TAG}/preview/coreml/rfdetr_keypoint_preview_coreml_fp16.pte`, + modelOpts: RFDETR_KEYPOINT_OPTS, +}; const RFDETR_KEYPOINT_COREML_FP32: KeypointDetectorModel<'xyxy', CocoLandmark> = { modelPath: `${BASE_URL}-rfdetr-keypoint/${VERSION_TAG}/preview/coreml/rfdetr_keypoint_preview_coreml_fp32.pte`, modelOpts: RFDETR_KEYPOINT_OPTS, @@ -1650,20 +1654,24 @@ export const models = { RFDETR_KEYPOINT: variants( { XNNPACK_FP32: RFDETR_KEYPOINT_XNNPACK_FP32, + COREML_FP16: RFDETR_KEYPOINT_COREML_FP16, COREML_FP32: RFDETR_KEYPOINT_COREML_FP32, MLX_FP32: RFDETR_KEYPOINT_MLX_FP32, }, // Core ML over MLX: 164.1 ms against 272.3 on an iPhone 16, at 378 MB - // against 1304 MB. Note this is not a Neural Engine win — the ANE is - // fp16-only, so this fp32 program never reaches it, and a compute-unit - // sweep confirms it (`cpu_and_ne` 337.5 ms matches `cpu_only` 352.9, - // while `all` 173.6 matches `cpu_and_gpu` 176.7). An fp16 build would - // reach the ANE and is 23% faster; exporting one is open, and blocked on - // the export environment rather than on precision. fp16 and fp32 built - // from the same trace agree to 0.0008 confidence on device, but both - // score 3x below the published fp32, and that build's `rfdetr` leaves - // `keypoint_head.keypoint_proj.*` unconsumed when loading the checkpoint. - { ios: 'COREML_FP32' } + // against 1304 MB. fp16 over fp32: 144.0 ms against 165.1, a 263 MB peak + // against 389 MB, and half the download, matching fp32 to 0.0008 + // confidence and 0.23 px. + // + // That fp16 build took a while to exist and is fragile in a specific way, + // so do not "simplify" it. It is correct on device only with both the CPU + // and the Neural Engine excluded — measured against fp32's 0.863, `all` + // gives 0.421, `cpu_only` 0.411, `cpu_and_ne` 0.422 and `cpu_and_gpu` + // 0.862 — and on macOS only the CPU path is wrong, so a host check passes + // a build the device gets wrong. It also needs an iOS17 deployment target; + // iOS18 degrades every build of this model at either precision. Both + // constraints live in export-scripts (MR !18). + { ios: 'COREML_FP16' } ), }, From 0565d9aaa2b7becdffe313d15e625cd7d0745afb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20S=C5=82uszniak?= Date: Fri, 28 Aug 2026 13:54:47 +0200 Subject: [PATCH 08/24] fix(models): point RF-DETR keypoint at v0.10.0, where its files actually are The previous commit added COREML_FP16 next to COREML_FP32 and inherited that line's VERSION_TAG (v0.9.0). The fp16 build was published to v0.10.0, and v0.9.0 holds only the three original fp32 files, so the URL 404s -- confirmed against the Hub, 404 at v0.9.0 and 206 at v0.10.0. Since the same commit made COREML_FP16 the iOS default, every iOS app resolving RFDETR_KEYPOINT.DEFAULT would have failed to download it. Moves all four keypoint URLs to NEXT_VERSION_TAG, which is the tag this branch targets and which carries xnnpack, coreml fp16, coreml fp32 and mlx. All four verified to resolve. rfdetr-nano-detector is the only entry still on VERSION_TAG; it is untouched here because nothing in this change republished it. --- packages/react-native-executorch/src/models.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts index 770ce4bc2d..355c29e139 100644 --- a/packages/react-native-executorch/src/models.ts +++ b/packages/react-native-executorch/src/models.ts @@ -487,19 +487,19 @@ const RFDETR_KEYPOINT_OPTS = { landmarks: COCO_LANDMARKS, }; const RFDETR_KEYPOINT_XNNPACK_FP32: KeypointDetectorModel<'xyxy', CocoLandmark> = { - modelPath: `${BASE_URL}-rfdetr-keypoint/${VERSION_TAG}/preview/xnnpack/rfdetr_keypoint_preview_xnnpack_fp32.pte`, + modelPath: `${BASE_URL}-rfdetr-keypoint/${NEXT_VERSION_TAG}/preview/xnnpack/rfdetr_keypoint_preview_xnnpack_fp32.pte`, modelOpts: RFDETR_KEYPOINT_OPTS, }; const RFDETR_KEYPOINT_COREML_FP16: KeypointDetectorModel<'xyxy', CocoLandmark> = { - modelPath: `${BASE_URL}-rfdetr-keypoint/${VERSION_TAG}/preview/coreml/rfdetr_keypoint_preview_coreml_fp16.pte`, + modelPath: `${BASE_URL}-rfdetr-keypoint/${NEXT_VERSION_TAG}/preview/coreml/rfdetr_keypoint_preview_coreml_fp16.pte`, modelOpts: RFDETR_KEYPOINT_OPTS, }; const RFDETR_KEYPOINT_COREML_FP32: KeypointDetectorModel<'xyxy', CocoLandmark> = { - modelPath: `${BASE_URL}-rfdetr-keypoint/${VERSION_TAG}/preview/coreml/rfdetr_keypoint_preview_coreml_fp32.pte`, + modelPath: `${BASE_URL}-rfdetr-keypoint/${NEXT_VERSION_TAG}/preview/coreml/rfdetr_keypoint_preview_coreml_fp32.pte`, modelOpts: RFDETR_KEYPOINT_OPTS, }; const RFDETR_KEYPOINT_MLX_FP32: KeypointDetectorModel<'xyxy', CocoLandmark> = { - modelPath: `${BASE_URL}-rfdetr-keypoint/${VERSION_TAG}/preview/mlx/rfdetr_keypoint_preview_mlx_fp32.pte`, + modelPath: `${BASE_URL}-rfdetr-keypoint/${NEXT_VERSION_TAG}/preview/mlx/rfdetr_keypoint_preview_mlx_fp32.pte`, modelOpts: RFDETR_KEYPOINT_OPTS, }; From 3e4365cf9856b111d123add89dbb68afda09e2f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20S=C5=82uszniak?= Date: Fri, 28 Aug 2026 13:58:31 +0200 Subject: [PATCH 09/24] feat(models): drop the RF-DETR keypoint fp32 Core ML build, repoint nano-detector fp32 was kept as a safety net for the fp16 build, which is correct on device only under CPU_AND_GPU. Having pinned and documented that, the net was worth what it cost only if fp16 were unproven, so it was checked properly: 13 real photos, 40 detections, fp16 against fp32 on each. Visible landmarks agree to 1.04 px, boxes to 0.653 px, scores to 0.072, and no detection crossed the confidence threshold. Landmarks the model marks invisible drift up to 15.7 px, but their coordinates are undefined when `vis` is 0. That leaves no argument for shipping 148 MB nobody downloads, so COREML_FP32 is gone and fp16 is the only Core ML build. The file is removed from the Hub as well; v0.9.0 keeps its own copy, so released versions are unaffected. Also repoints rfdetr-nano-detector's xnnpack URL at NEXT_VERSION_TAG. That entry was split across tags, xnnpack on v0.9.0 and coreml on v0.10.0, for no reason; the file exists at both, so this only makes it consistent. --- .../react-native-executorch/src/models.ts | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts index 355c29e139..0e76021b0c 100644 --- a/packages/react-native-executorch/src/models.ts +++ b/packages/react-native-executorch/src/models.ts @@ -296,7 +296,7 @@ const RFDETR_NANO_DETECTOR_OPTS = { defaultIouThreshold: 0.55, }; const RFDETR_NANO_DETECTOR_XNNPACK_FP32: ObjectDetectorModel<'xyxy', CocoClass> = { - modelPath: `${BASE_URL}-rfdetr-nano-detector/${VERSION_TAG}/xnnpack/rfdetr_nano_xnnpack_fp32.pte`, + modelPath: `${BASE_URL}-rfdetr-nano-detector/${NEXT_VERSION_TAG}/xnnpack/rfdetr_nano_xnnpack_fp32.pte`, modelOpts: RFDETR_NANO_DETECTOR_OPTS, }; const RFDETR_NANO_DETECTOR_COREML_FP16: ObjectDetectorModel<'xyxy', CocoClass> = { @@ -494,10 +494,6 @@ const RFDETR_KEYPOINT_COREML_FP16: KeypointDetectorModel<'xyxy', CocoLandmark> = modelPath: `${BASE_URL}-rfdetr-keypoint/${NEXT_VERSION_TAG}/preview/coreml/rfdetr_keypoint_preview_coreml_fp16.pte`, modelOpts: RFDETR_KEYPOINT_OPTS, }; -const RFDETR_KEYPOINT_COREML_FP32: KeypointDetectorModel<'xyxy', CocoLandmark> = { - modelPath: `${BASE_URL}-rfdetr-keypoint/${NEXT_VERSION_TAG}/preview/coreml/rfdetr_keypoint_preview_coreml_fp32.pte`, - modelOpts: RFDETR_KEYPOINT_OPTS, -}; const RFDETR_KEYPOINT_MLX_FP32: KeypointDetectorModel<'xyxy', CocoLandmark> = { modelPath: `${BASE_URL}-rfdetr-keypoint/${NEXT_VERSION_TAG}/preview/mlx/rfdetr_keypoint_preview_mlx_fp32.pte`, modelOpts: RFDETR_KEYPOINT_OPTS, @@ -1655,13 +1651,18 @@ export const models = { { XNNPACK_FP32: RFDETR_KEYPOINT_XNNPACK_FP32, COREML_FP16: RFDETR_KEYPOINT_COREML_FP16, - COREML_FP32: RFDETR_KEYPOINT_COREML_FP32, MLX_FP32: RFDETR_KEYPOINT_MLX_FP32, }, - // Core ML over MLX: 164.1 ms against 272.3 on an iPhone 16, at 378 MB - // against 1304 MB. fp16 over fp32: 144.0 ms against 165.1, a 263 MB peak - // against 389 MB, and half the download, matching fp32 to 0.0008 - // confidence and 0.23 px. + // Core ML over MLX: 144.0 ms against 272.3 on an iPhone 16, at 263 MB + // against 1304 MB. + // + // fp16 is the only Core ML build published for this model. It replaced an + // fp32 one that was 165.1 ms, 389 MB peak and twice the download, after + // fp16 was checked against it across 13 real photos and 40 detections: + // visible landmarks agreed to 1.04 px, boxes to 0.653 px and scores to + // 0.072, with no detection crossing the threshold. (Landmarks the model + // marks invisible drift further, up to 15.7 px, but their coordinates are + // undefined when `vis` is 0.) // // That fp16 build took a while to exist and is fragile in a specific way, // so do not "simplify" it. It is correct on device only with both the CPU From 776f51597290b88df8aa37de3e15928bf9395d26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20S=C5=82uszniak?= Date: Fri, 28 Aug 2026 13:59:03 +0200 Subject: [PATCH 10/24] fix(computer-vision): the keypoint demo still offered the removed fp32 build Missed in the previous commit: the demo's model picker referenced RFDETR_KEYPOINT.COREML_FP32, which no longer exists. Points it at COREML_FP16. --- apps/computer-vision/app/keypoint/index.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/computer-vision/app/keypoint/index.tsx b/apps/computer-vision/app/keypoint/index.tsx index cc6fce3309..377d502666 100644 --- a/apps/computer-vision/app/keypoint/index.tsx +++ b/apps/computer-vision/app/keypoint/index.tsx @@ -27,8 +27,8 @@ const MODEL_OPTIONS: ModelOption[] = [ value: models.keypointDetection.RFDETR_KEYPOINT.XNNPACK_FP32, }, { - label: 'RF-DETR Keypoint (CoreML FP32)', - value: models.keypointDetection.RFDETR_KEYPOINT.COREML_FP32, + label: 'RF-DETR Keypoint (CoreML FP16)', + value: models.keypointDetection.RFDETR_KEYPOINT.COREML_FP16, disabled: Platform.OS !== 'ios', }, { From ce6fbb7cf16c0483ddd991b9db9b7b52e00deb12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20S=C5=82uszniak?= Date: Fri, 28 Aug 2026 15:13:20 +0200 Subject: [PATCH 11/24] feat(models): wire every published variant the registry was missing An audit of the 48 HF repos the registry references found 41 published .pte files no app could reach: the entry simply was not there. That matters most for the accelerated ones, because `DEFAULT` prefers Core ML on iOS and Vulkan on Android, so a missing key silently pins those models to XNNPACK forever. 38 of the 41 are now wired: - Core ML for all-MiniLM-L6-v2, multi-qa-MiniLM, paraphrase-multilingual, distiluse and CLIP text - Core ML for YOLO26-seg (5 scales x 3 input sizes) and YOLO26-pose - Core ML for Kokoro, all three model sets, reached by all 9 languages - MLX int4/int8 for LFM2.5-VL 1.6B, Vulkan for Gemma 4 E2B - XNNPACK fp32 for paraphrase, distiluse and PP-OCRv6, XNNPACK int8 for the three English Whisper sizes Left out: `all-MiniLM-L6-v2_xnnpack.pte`, a pre-rename duplicate of the file the registry already uses, and the two Llama 3.2 QAT+LoRA builds, which the rewrite does not support. New XNNPACK variants are declared after the incumbent so no CPU default moves. Accelerated ones take the platform default, per the rule that a published accelerated export is the signal to prefer it: 32 groups move on iOS, 1 on Android. Two calls that rule does not make on its own: Kokoro's Core ML files are fp32, the registry's only Core ML fp32 now that RF-DETR keypoint's is gone, and fp32 never reaches the Neural Engine. They still default because the duration predictor runs 14-21x faster warm on an iPhone 16; the first-use compile cost is in the comment. distiluse is the only Core ML/MLX pair that does not go Core ML's way, so it is pinned to MLX. Measured on an iPhone 16 over 150 warm embeds of five sentences in five languages: MLX int8 3.46 ms, Core ML fp16 3.96 ms, XNNPACK 8da4w 5.86 ms, holding when the arms are reversed. MLX is half the download and skips a 786 ms first-use compile, and fidelity does not break the tie (both 0.973 worst-case cosine against XNNPACK). That pin is why `defaults to Core ML on iOS` now exempts groups carrying an explicit iOS pin: Core ML leads the order, but the order was never a benchmark result. Groups with both backends still have to pin one. `download-libs.js` gains coreml for textEmbeddings and textToSpeech and vulkan for llm, without which those defaults could never resolve. --- .../__tests__/api/modelVariants.test.ts | 12 +- .../scripts/download-libs.js | 14 +- .../react-native-executorch/src/models.ts | 245 +++++++++++++++++- 3 files changed, 257 insertions(+), 14 deletions(-) diff --git a/packages/react-native-executorch/__tests__/api/modelVariants.test.ts b/packages/react-native-executorch/__tests__/api/modelVariants.test.ts index e33e54ab5a..8a93f6fb4a 100644 --- a/packages/react-native-executorch/__tests__/api/modelVariants.test.ts +++ b/packages/react-native-executorch/__tests__/api/modelVariants.test.ts @@ -139,8 +139,18 @@ describe('DEFAULT variant resolution', () => { expect(defaultsOf(registryFor({ os: 'ios' })).length).toBeGreaterThan(100); }); - it('defaults to Core ML on iOS wherever a Core ML export exists', () => { + it('defaults to Core ML on iOS wherever a Core ML export exists and nothing is pinned', () => { + // Core ML leads the iOS order, so it wins by default. It does not always + // win on the device: distiluse pins MLX because MLX measured faster there. + // A group that names an explicit iOS pin has been measured, so it is + // exempt; the pin itself is guarded by `pins one` below. + const pinned = new Set( + variantsCalls() + .filter(({ pinsIos }) => pinsIos) + .map(({ name }) => name) + ); const offenders = defaultsOf(registryFor({ os: 'ios' })) + .filter(({ label }) => !pinned.has(label.split('.').pop()!)) .filter(({ offers }) => offers.some((key) => key.startsWith('COREML'))) .filter(({ key }) => key !== undefined && !key.startsWith('COREML')) .map(({ label, key }) => `${label}: ${key}`); diff --git a/packages/react-native-executorch/scripts/download-libs.js b/packages/react-native-executorch/scripts/download-libs.js index 4e6b75af7a..e5f9077ae9 100644 --- a/packages/react-native-executorch/scripts/download-libs.js +++ b/packages/react-native-executorch/scripts/download-libs.js @@ -121,8 +121,9 @@ const ALL_LIBS = ['opencv', 'phonemis']; // downloaded silently drops that model back to XNNPACK. The registry test // `feature map` in __tests__/api/modelVariants.test.ts holds this in sync. const FEATURE_MAP = { - // Text-only LLMs ship xnnpack + mlx (Gemma 4 ships an MLX iOS export). - llm: { backends: ['xnnpack', 'mlx'], libs: [] }, + // Text-only LLMs ship xnnpack + mlx (Gemma 4 ships an MLX iOS export) and + // vulkan (Gemma 4 E2B ships a Vulkan export). + llm: { backends: ['xnnpack', 'mlx', 'vulkan'], libs: [] }, // Multimodal LLMs add vulkan (Gemma-3-multimodal ships a Vulkan export) and // mlx (Gemma 4 ships an MLX iOS export); the vision encoder needs opencv. multimodalLLM: { backends: ['xnnpack', 'mlx', 'vulkan'], libs: ['opencv'] }, @@ -130,12 +131,13 @@ const FEATURE_MAP = { privacyFilter: { backends: ['xnnpack', 'mlx'], libs: [] }, // Whisper ships xnnpack, coreml and an MLX iOS export. speechToText: { backends: ['xnnpack', 'coreml', 'mlx'], libs: [] }, - // Kokoro ships xnnpack; Supertonic adds an MLX iOS export. - textToSpeech: { backends: ['xnnpack', 'mlx'], libs: ['phonemis'] }, + // Kokoro ships xnnpack + coreml; Supertonic adds an MLX iOS export. + textToSpeech: { backends: ['xnnpack', 'coreml', 'mlx'], libs: ['phonemis'] }, // FSMN VAD — xnnpack only. vad: { backends: ['xnnpack'], libs: [] }, - // LFM2.5-Embedding ships an MLX iOS export alongside xnnpack. - textEmbeddings: { backends: ['xnnpack', 'mlx'], libs: [] }, + // The MiniLM/CLIP-text/distiluse family ships coreml alongside xnnpack, and + // LFM2.5-Embedding and distiluse add MLX iOS exports. + textEmbeddings: { backends: ['xnnpack', 'coreml', 'mlx'], libs: [] }, // CLIP's vision encoder ships xnnpack, coreml and mlx. imageEmbeddings: { backends: ['xnnpack', 'coreml', 'mlx'], libs: ['opencv'] }, // EfficientNet ships xnnpack + coreml. diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts index 0e76021b0c..cf26227507 100644 --- a/packages/react-native-executorch/src/models.ts +++ b/packages/react-native-executorch/src/models.ts @@ -468,14 +468,26 @@ const YOLO26_POSE_384_XNNPACK_FP32: KeypointDetectorModel<'xyxy', CocoLandmark> modelPath: `${BASE_URL}-yolo26-pose/${NEXT_VERSION_TAG}/xnnpack/yolo26n_pose_384_xnnpack_fp32.pte`, modelOpts: YOLO26_POSE_OPTS, }; +const YOLO26_POSE_384_COREML_FP16: KeypointDetectorModel<'xyxy', CocoLandmark> = { + modelPath: `${BASE_URL}-yolo26-pose/${NEXT_VERSION_TAG}/coreml/yolo26n_pose_384_coreml_fp16.pte`, + modelOpts: YOLO26_POSE_OPTS, +}; const YOLO26_POSE_512_XNNPACK_FP32: KeypointDetectorModel<'xyxy', CocoLandmark> = { modelPath: `${BASE_URL}-yolo26-pose/${NEXT_VERSION_TAG}/xnnpack/yolo26n_pose_512_xnnpack_fp32.pte`, modelOpts: YOLO26_POSE_OPTS, }; +const YOLO26_POSE_512_COREML_FP16: KeypointDetectorModel<'xyxy', CocoLandmark> = { + modelPath: `${BASE_URL}-yolo26-pose/${NEXT_VERSION_TAG}/coreml/yolo26n_pose_512_coreml_fp16.pte`, + modelOpts: YOLO26_POSE_OPTS, +}; const YOLO26_POSE_640_XNNPACK_FP32: KeypointDetectorModel<'xyxy', CocoLandmark> = { modelPath: `${BASE_URL}-yolo26-pose/${NEXT_VERSION_TAG}/xnnpack/yolo26n_pose_640_xnnpack_fp32.pte`, modelOpts: YOLO26_POSE_OPTS, }; +const YOLO26_POSE_640_COREML_FP16: KeypointDetectorModel<'xyxy', CocoLandmark> = { + modelPath: `${BASE_URL}-yolo26-pose/${NEXT_VERSION_TAG}/coreml/yolo26n_pose_640_coreml_fp16.pte`, + modelOpts: YOLO26_POSE_OPTS, +}; const RFDETR_KEYPOINT_OPTS = { boxFormat: 'xyxy' as const, @@ -563,66 +575,126 @@ const YOLO26_NANO_SEG_384_XNNPACK_FP32: InstanceSegmenterModel<'xyxy', CocoClass modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/n/xnnpack/yolo26_seg_n_384_xnnpack_fp32.pte`, modelOpts: YOLO26_SEG_OPTS, }; +const YOLO26_NANO_SEG_384_COREML_FP16: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { + modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/n/coreml/yolo26_seg_n_384_coreml_fp16.pte`, + modelOpts: YOLO26_SEG_OPTS, +}; const YOLO26_NANO_SEG_512_XNNPACK_FP32: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/n/xnnpack/yolo26_seg_n_512_xnnpack_fp32.pte`, modelOpts: YOLO26_SEG_OPTS, }; +const YOLO26_NANO_SEG_512_COREML_FP16: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { + modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/n/coreml/yolo26_seg_n_512_coreml_fp16.pte`, + modelOpts: YOLO26_SEG_OPTS, +}; const YOLO26_NANO_SEG_640_XNNPACK_FP32: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/n/xnnpack/yolo26_seg_n_640_xnnpack_fp32.pte`, modelOpts: YOLO26_SEG_OPTS, }; +const YOLO26_NANO_SEG_640_COREML_FP16: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { + modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/n/coreml/yolo26_seg_n_640_coreml_fp16.pte`, + modelOpts: YOLO26_SEG_OPTS, +}; const YOLO26_SMALL_SEG_384_XNNPACK_FP32: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/s/xnnpack/yolo26_seg_s_384_xnnpack_fp32.pte`, modelOpts: YOLO26_SEG_OPTS, }; +const YOLO26_SMALL_SEG_384_COREML_FP16: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { + modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/s/coreml/yolo26_seg_s_384_coreml_fp16.pte`, + modelOpts: YOLO26_SEG_OPTS, +}; const YOLO26_SMALL_SEG_512_XNNPACK_FP32: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/s/xnnpack/yolo26_seg_s_512_xnnpack_fp32.pte`, modelOpts: YOLO26_SEG_OPTS, }; +const YOLO26_SMALL_SEG_512_COREML_FP16: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { + modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/s/coreml/yolo26_seg_s_512_coreml_fp16.pte`, + modelOpts: YOLO26_SEG_OPTS, +}; const YOLO26_SMALL_SEG_640_XNNPACK_FP32: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/s/xnnpack/yolo26_seg_s_640_xnnpack_fp32.pte`, modelOpts: YOLO26_SEG_OPTS, }; +const YOLO26_SMALL_SEG_640_COREML_FP16: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { + modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/s/coreml/yolo26_seg_s_640_coreml_fp16.pte`, + modelOpts: YOLO26_SEG_OPTS, +}; const YOLO26_MEDIUM_SEG_384_XNNPACK_FP32: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/m/xnnpack/yolo26_seg_m_384_xnnpack_fp32.pte`, modelOpts: YOLO26_SEG_OPTS, }; +const YOLO26_MEDIUM_SEG_384_COREML_FP16: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { + modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/m/coreml/yolo26_seg_m_384_coreml_fp16.pte`, + modelOpts: YOLO26_SEG_OPTS, +}; const YOLO26_MEDIUM_SEG_512_XNNPACK_FP32: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/m/xnnpack/yolo26_seg_m_512_xnnpack_fp32.pte`, modelOpts: YOLO26_SEG_OPTS, }; +const YOLO26_MEDIUM_SEG_512_COREML_FP16: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { + modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/m/coreml/yolo26_seg_m_512_coreml_fp16.pte`, + modelOpts: YOLO26_SEG_OPTS, +}; const YOLO26_MEDIUM_SEG_640_XNNPACK_FP32: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/m/xnnpack/yolo26_seg_m_640_xnnpack_fp32.pte`, modelOpts: YOLO26_SEG_OPTS, }; +const YOLO26_MEDIUM_SEG_640_COREML_FP16: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { + modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/m/coreml/yolo26_seg_m_640_coreml_fp16.pte`, + modelOpts: YOLO26_SEG_OPTS, +}; const YOLO26_LARGE_SEG_384_XNNPACK_FP32: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/l/xnnpack/yolo26_seg_l_384_xnnpack_fp32.pte`, modelOpts: YOLO26_SEG_OPTS, }; +const YOLO26_LARGE_SEG_384_COREML_FP16: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { + modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/l/coreml/yolo26_seg_l_384_coreml_fp16.pte`, + modelOpts: YOLO26_SEG_OPTS, +}; const YOLO26_LARGE_SEG_512_XNNPACK_FP32: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/l/xnnpack/yolo26_seg_l_512_xnnpack_fp32.pte`, modelOpts: YOLO26_SEG_OPTS, }; +const YOLO26_LARGE_SEG_512_COREML_FP16: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { + modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/l/coreml/yolo26_seg_l_512_coreml_fp16.pte`, + modelOpts: YOLO26_SEG_OPTS, +}; const YOLO26_LARGE_SEG_640_XNNPACK_FP32: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/l/xnnpack/yolo26_seg_l_640_xnnpack_fp32.pte`, modelOpts: YOLO26_SEG_OPTS, }; +const YOLO26_LARGE_SEG_640_COREML_FP16: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { + modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/l/coreml/yolo26_seg_l_640_coreml_fp16.pte`, + modelOpts: YOLO26_SEG_OPTS, +}; const YOLO26_XLARGE_SEG_384_XNNPACK_FP32: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/x/xnnpack/yolo26_seg_x_384_xnnpack_fp32.pte`, modelOpts: YOLO26_SEG_OPTS, }; +const YOLO26_XLARGE_SEG_384_COREML_FP16: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { + modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/x/coreml/yolo26_seg_x_384_coreml_fp16.pte`, + modelOpts: YOLO26_SEG_OPTS, +}; const YOLO26_XLARGE_SEG_512_XNNPACK_FP32: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/x/xnnpack/yolo26_seg_x_512_xnnpack_fp32.pte`, modelOpts: YOLO26_SEG_OPTS, }; +const YOLO26_XLARGE_SEG_512_COREML_FP16: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { + modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/x/coreml/yolo26_seg_x_512_coreml_fp16.pte`, + modelOpts: YOLO26_SEG_OPTS, +}; const YOLO26_XLARGE_SEG_640_XNNPACK_FP32: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/x/xnnpack/yolo26_seg_x_640_xnnpack_fp32.pte`, modelOpts: YOLO26_SEG_OPTS, }; +const YOLO26_XLARGE_SEG_640_COREML_FP16: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { + modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/x/coreml/yolo26_seg_x_640_coreml_fp16.pte`, + modelOpts: YOLO26_SEG_OPTS, +}; // ============================================================================= // Text Embeddings @@ -631,6 +703,10 @@ const ALL_MINILM_L6_V2_EMBEDDINGS: TextEmbedderModel = { modelPath: `${BASE_URL}-all-MiniLM-L6-v2/${NEXT_VERSION_TAG}/xnnpack/all_minilm_l6_v2_xnnpack_fp32.pte`, tokenizerPath: `${BASE_URL}-all-MiniLM-L6-v2/${NEXT_VERSION_TAG}/tokenizer.json`, }; +const ALL_MINILM_L6_V2_COREML_FP16: TextEmbedderModel = { + modelPath: `${BASE_URL}-all-MiniLM-L6-v2/${NEXT_VERSION_TAG}/coreml/all_minilm_l6_v2_coreml_fp16.pte`, + tokenizerPath: `${BASE_URL}-all-MiniLM-L6-v2/${NEXT_VERSION_TAG}/tokenizer.json`, +}; const ALL_MPNET_BASE_V2_EMBEDDINGS: TextEmbedderModel = { modelPath: `${BASE_URL}-all-mpnet-base-v2/${NEXT_VERSION_TAG}/xnnpack/all_mpnet_base_v2_xnnpack_fp32.pte`, tokenizerPath: `${BASE_URL}-all-mpnet-base-v2/${NEXT_VERSION_TAG}/tokenizer.json`, @@ -639,6 +715,10 @@ const MULTI_QA_MINILM_L6_COS_V1_EMBEDDINGS: TextEmbedderModel = { modelPath: `${BASE_URL}-multi-qa-MiniLM-L6-cos-v1/${NEXT_VERSION_TAG}/xnnpack/multi_qa_minilm_l6_cos_v1_xnnpack_fp32.pte`, tokenizerPath: `${BASE_URL}-multi-qa-MiniLM-L6-cos-v1/${NEXT_VERSION_TAG}/tokenizer.json`, }; +const MULTI_QA_MINILM_L6_COS_V1_COREML_FP16: TextEmbedderModel = { + modelPath: `${BASE_URL}-multi-qa-MiniLM-L6-cos-v1/${NEXT_VERSION_TAG}/coreml/multi_qa_minilm_l6_cos_v1_coreml_fp16.pte`, + tokenizerPath: `${BASE_URL}-multi-qa-MiniLM-L6-cos-v1/${NEXT_VERSION_TAG}/tokenizer.json`, +}; const MULTI_QA_MPNET_BASE_DOT_V1_EMBEDDINGS: TextEmbedderModel = { modelPath: `${BASE_URL}-multi-qa-mpnet-base-dot-v1/${NEXT_VERSION_TAG}/xnnpack/multi_qa_mpnet_base_dot_v1_xnnpack_fp32.pte`, tokenizerPath: `${BASE_URL}-multi-qa-mpnet-base-dot-v1/${NEXT_VERSION_TAG}/tokenizer.json`, @@ -647,10 +727,26 @@ const PARAPHRASE_MULTILINGUAL_MINILM_L12_V2_EMBEDDINGS: TextEmbedderModel = { modelPath: `${BASE_URL}-paraphrase-multilingual-MiniLM-L12-v2/${NEXT_VERSION_TAG}/xnnpack/paraphrase_multilingual_minilm_l12_v2_xnnpack_8da4w.pte`, tokenizerPath: `${BASE_URL}-paraphrase-multilingual-MiniLM-L12-v2/${NEXT_VERSION_TAG}/tokenizer.json`, }; +const PARAPHRASE_MULTILINGUAL_MINILM_L12_V2_XNNPACK_FP32: TextEmbedderModel = { + modelPath: `${BASE_URL}-paraphrase-multilingual-MiniLM-L12-v2/${NEXT_VERSION_TAG}/xnnpack/paraphrase_multilingual_minilm_l12_v2_xnnpack_fp32.pte`, + tokenizerPath: `${BASE_URL}-paraphrase-multilingual-MiniLM-L12-v2/${NEXT_VERSION_TAG}/tokenizer.json`, +}; +const PARAPHRASE_MULTILINGUAL_MINILM_L12_V2_COREML_FP16: TextEmbedderModel = { + modelPath: `${BASE_URL}-paraphrase-multilingual-MiniLM-L12-v2/${NEXT_VERSION_TAG}/coreml/paraphrase_multilingual_minilm_l12_v2_coreml_fp16.pte`, + tokenizerPath: `${BASE_URL}-paraphrase-multilingual-MiniLM-L12-v2/${NEXT_VERSION_TAG}/tokenizer.json`, +}; const DISTILUSE_BASE_MULTILINGUAL_CASED_V2_EMBEDDINGS: TextEmbedderModel = { modelPath: `${BASE_URL}-distiluse-base-multilingual-cased-v2/${NEXT_VERSION_TAG}/xnnpack/distiluse_base_multilingual_cased_v2_xnnpack_8da4w.pte`, tokenizerPath: `${BASE_URL}-distiluse-base-multilingual-cased-v2/${NEXT_VERSION_TAG}/tokenizer.json`, }; +const DISTILUSE_BASE_MULTILINGUAL_CASED_V2_XNNPACK_FP32: TextEmbedderModel = { + modelPath: `${BASE_URL}-distiluse-base-multilingual-cased-v2/${NEXT_VERSION_TAG}/xnnpack/distiluse_base_multilingual_cased_v2_xnnpack_fp32.pte`, + tokenizerPath: `${BASE_URL}-distiluse-base-multilingual-cased-v2/${NEXT_VERSION_TAG}/tokenizer.json`, +}; +const DISTILUSE_BASE_MULTILINGUAL_CASED_V2_COREML_FP16: TextEmbedderModel = { + modelPath: `${BASE_URL}-distiluse-base-multilingual-cased-v2/${NEXT_VERSION_TAG}/coreml/distiluse_base_multilingual_cased_v2_coreml_fp16.pte`, + tokenizerPath: `${BASE_URL}-distiluse-base-multilingual-cased-v2/${NEXT_VERSION_TAG}/tokenizer.json`, +}; const DISTILUSE_BASE_MULTILINGUAL_CASED_V2_MLX_INT8: TextEmbedderModel = { modelPath: `${BASE_URL}-distiluse-base-multilingual-cased-v2/${NEXT_VERSION_TAG}/mlx/distiluse_base_multilingual_cased_v2_mlx_int8.pte`, tokenizerPath: `${BASE_URL}-distiluse-base-multilingual-cased-v2/${NEXT_VERSION_TAG}/tokenizer.json`, @@ -659,6 +755,10 @@ const CLIP_VIT_BASE_PATCH32_TEXT_EMBEDDINGS: TextEmbedderModel = { modelPath: `${BASE_URL}-clip-vit-base-patch32/${NEXT_VERSION_TAG}/xnnpack/clip_vit_base_patch32_text_xnnpack_fp32.pte`, tokenizerPath: `${BASE_URL}-clip-vit-base-patch32/${NEXT_VERSION_TAG}/tokenizer.json`, }; +const CLIP_VIT_BASE_PATCH32_TEXT_COREML_FP16: TextEmbedderModel = { + modelPath: `${BASE_URL}-clip-vit-base-patch32/${NEXT_VERSION_TAG}/coreml/clip_vit_base_patch32_text_coreml_fp16.pte`, + tokenizerPath: `${BASE_URL}-clip-vit-base-patch32/${NEXT_VERSION_TAG}/tokenizer.json`, +}; const LFM2_5_EMBEDDING_350M_EMBEDDINGS: TextEmbedderModel = { modelPath: `${BASE_URL}-lfm2.5-embedding-350m/${NEXT_VERSION_TAG}/xnnpack/lfm_2_5_embedding_350m_xnnpack_8da4w.pte`, tokenizerPath: `${BASE_URL}-lfm2.5-embedding-350m/${NEXT_VERSION_TAG}/tokenizer.json`, @@ -714,6 +814,12 @@ const WHISPER_TINY_EN_XNNPACK_FP32: WhisperSttModel<'en'> = { supportedLanguages: ['en'], vadModel: FSMN_VAD_XNNPACK_FP32, }; +const WHISPER_TINY_EN_XNNPACK_INT8: WhisperSttModel<'en'> = { + modelPath: `${BASE_URL}-whisper-tiny.en/${NEXT_VERSION_TAG}/xnnpack/whisper_tiny_en_xnnpack_int8.pte`, + tokenizerPath: `${BASE_URL}-whisper-tiny.en/${NEXT_VERSION_TAG}/tokenizer.json`, + supportedLanguages: ['en'], + vadModel: FSMN_VAD_XNNPACK_FP32, +}; const WHISPER_TINY_EN_COREML_FP16: WhisperSttModel<'en'> = { modelPath: `${BASE_URL}-whisper-tiny.en/${NEXT_VERSION_TAG}/coreml/whisper_tiny_en_coreml_fp16.pte`, tokenizerPath: `${BASE_URL}-whisper-tiny.en/${NEXT_VERSION_TAG}/tokenizer.json`, @@ -764,6 +870,12 @@ const WHISPER_BASE_EN_XNNPACK_FP32: WhisperSttModel<'en'> = { supportedLanguages: ['en'], vadModel: FSMN_VAD_XNNPACK_FP32, }; +const WHISPER_BASE_EN_XNNPACK_INT8: WhisperSttModel<'en'> = { + modelPath: `${BASE_URL}-whisper-base.en/${NEXT_VERSION_TAG}/xnnpack/whisper_base_en_xnnpack_int8.pte`, + tokenizerPath: `${BASE_URL}-whisper-base.en/${NEXT_VERSION_TAG}/tokenizer.json`, + supportedLanguages: ['en'], + vadModel: FSMN_VAD_XNNPACK_FP32, +}; const WHISPER_BASE_EN_COREML_FP16: WhisperSttModel<'en'> = { modelPath: `${BASE_URL}-whisper-base.en/${NEXT_VERSION_TAG}/coreml/whisper_base_en_coreml_fp16.pte`, tokenizerPath: `${BASE_URL}-whisper-base.en/${NEXT_VERSION_TAG}/tokenizer.json`, @@ -814,6 +926,12 @@ const WHISPER_SMALL_EN_XNNPACK_FP32: WhisperSttModel<'en'> = { supportedLanguages: ['en'], vadModel: FSMN_VAD_XNNPACK_FP32, }; +const WHISPER_SMALL_EN_XNNPACK_INT8: WhisperSttModel<'en'> = { + modelPath: `${BASE_URL}-whisper-small.en/${NEXT_VERSION_TAG}/xnnpack/whisper_small_en_xnnpack_int8.pte`, + tokenizerPath: `${BASE_URL}-whisper-small.en/${NEXT_VERSION_TAG}/tokenizer.json`, + supportedLanguages: ['en'], + vadModel: FSMN_VAD_XNNPACK_FP32, +}; const WHISPER_SMALL_EN_COREML_FP16: WhisperSttModel<'en'> = { modelPath: `${BASE_URL}-whisper-small.en/${NEXT_VERSION_TAG}/coreml/whisper_small_en_coreml_fp16.pte`, tokenizerPath: `${BASE_URL}-whisper-small.en/${NEXT_VERSION_TAG}/tokenizer.json`, @@ -897,14 +1015,29 @@ const SUPERTONIC_3_MLX_FP32: SupertonicTtsModel = { const KOKORO_ROOT = `${BASE_URL}-kokoro/${NEXT_VERSION_TAG}`; const KOKORO_PHONEMIZER_ROOT = `${KOKORO_ROOT}/phonemizer`; -const kokoroModelPaths = (variant: 'std' | 'pl' | 'de', dir: string) => ({ - durationPredictor: `${KOKORO_ROOT}/xnnpack/${dir}/duration_predictor_${variant}_xnnpack_fp32.pte`, - synthesizer: `${KOKORO_ROOT}/xnnpack/${dir}/synthesizer_${variant}_xnnpack_fp32.pte`, +// Both backends export at fp32, so the precision is not a parameter here. +const kokoroModelPaths = ( + variant: 'std' | 'pl' | 'de', + dir: string, + backend: 'xnnpack' | 'coreml' = 'xnnpack' +) => ({ + durationPredictor: `${KOKORO_ROOT}/${backend}/${dir}/duration_predictor_${variant}_${backend}_fp32.pte`, + synthesizer: `${KOKORO_ROOT}/${backend}/${dir}/synthesizer_${variant}_${backend}_fp32.pte`, }); const KOKORO_STANDARD_PATHS = kokoroModelPaths('std', 'standard'); const KOKORO_POLISH_PATHS = kokoroModelPaths('pl', 'polish'); const KOKORO_GERMAN_PATHS = kokoroModelPaths('de', 'german'); +const KOKORO_STANDARD_COREML_PATHS = kokoroModelPaths('std', 'standard', 'coreml'); +const KOKORO_POLISH_COREML_PATHS = kokoroModelPaths('pl', 'polish', 'coreml'); +const KOKORO_GERMAN_COREML_PATHS = kokoroModelPaths('de', 'german', 'coreml'); + +// A language differs from its Core ML twin only in which pair of .pte files it +// loads: the voices, the phonemizer and the voice union are all backend-free. +const kokoroCoreMl = ( + model: KokoroTtsModel, + modelPaths: KokoroTtsModel['modelPaths'] +): KokoroTtsModel => ({ ...model, modelPaths }); const kokoroVoices = (names: readonly N[]) => names.reduce( @@ -985,6 +1118,22 @@ const KOKORO_DE_XNNPACK_FP32: KokoroTtsModel<'df_anna'> = { voices: kokoroVoices(['df_anna']), }; +const KOKORO_EN_US_COREML_FP32 = kokoroCoreMl( + KOKORO_EN_US_XNNPACK_FP32, + KOKORO_STANDARD_COREML_PATHS +); +const KOKORO_EN_GB_COREML_FP32 = kokoroCoreMl( + KOKORO_EN_GB_XNNPACK_FP32, + KOKORO_STANDARD_COREML_PATHS +); +const KOKORO_ES_COREML_FP32 = kokoroCoreMl(KOKORO_ES_XNNPACK_FP32, KOKORO_STANDARD_COREML_PATHS); +const KOKORO_FR_COREML_FP32 = kokoroCoreMl(KOKORO_FR_XNNPACK_FP32, KOKORO_STANDARD_COREML_PATHS); +const KOKORO_IT_COREML_FP32 = kokoroCoreMl(KOKORO_IT_XNNPACK_FP32, KOKORO_STANDARD_COREML_PATHS); +const KOKORO_PT_COREML_FP32 = kokoroCoreMl(KOKORO_PT_XNNPACK_FP32, KOKORO_STANDARD_COREML_PATHS); +const KOKORO_HI_COREML_FP32 = kokoroCoreMl(KOKORO_HI_XNNPACK_FP32, KOKORO_STANDARD_COREML_PATHS); +const KOKORO_PL_COREML_FP32 = kokoroCoreMl(KOKORO_PL_XNNPACK_FP32, KOKORO_POLISH_COREML_PATHS); +const KOKORO_DE_COREML_FP32 = kokoroCoreMl(KOKORO_DE_XNNPACK_FP32, KOKORO_GERMAN_COREML_PATHS); + // ============================================================================= // Privacy Filter // ============================================================================= @@ -1046,6 +1195,11 @@ const PPOCRV6_SMALL_XNNPACK_INT8: PaddleOcrModel = { charsetPath: PPOCRV6_CHARSET, modelOpts: PPOCRV6_OPTS, }; +const PPOCRV6_SMALL_XNNPACK_FP32: PaddleOcrModel = { + modelPath: `${BASE_URL}-pp-ocrv6/${NEXT_VERSION_TAG}/xnnpack/pp_ocrv6_xnnpack_fp32.pte`, + charsetPath: PPOCRV6_CHARSET, + modelOpts: PPOCRV6_OPTS, +}; const PPOCRV6_SMALL_COREML_INT8: PaddleOcrModel = { modelPath: `${BASE_URL}-pp-ocrv6/${NEXT_VERSION_TAG}/coreml/pp_ocrv6_coreml_int8.pte`, charsetPath: PPOCRV6_CHARSET, @@ -1139,6 +1293,20 @@ const LFM2_5_VL_1_6B_VULKAN_8DA4W: LLMModel = { modalities: ['image'], preprocessorConfig: LFM2_5_VL_PREPROCESSOR_CONFIG, }; +const LFM2_5_VL_1_6B_MLX_INT4: LLMModel = { + modelPath: `${LFM2_5_BASE_URL}/vl_1_6b/mlx/lfm_2_5_vl_1_6b_mlx_int4.pte`, + tokenizerPath: `${LFM2_5_BASE_URL}/vl_1_6b/tokenizer.json`, + tokenizerConfigPath: `${LFM2_5_BASE_URL}/vl_1_6b/tokenizer_config.json`, + modalities: ['image'], + preprocessorConfig: LFM2_5_VL_PREPROCESSOR_CONFIG, +}; +const LFM2_5_VL_1_6B_MLX_INT8: LLMModel = { + modelPath: `${LFM2_5_BASE_URL}/vl_1_6b/mlx/lfm_2_5_vl_1_6b_mlx_int8.pte`, + tokenizerPath: `${LFM2_5_BASE_URL}/vl_1_6b/tokenizer.json`, + tokenizerConfigPath: `${LFM2_5_BASE_URL}/vl_1_6b/tokenizer_config.json`, + modalities: ['image'], + preprocessorConfig: LFM2_5_VL_PREPROCESSOR_CONFIG, +}; const BIELIK_V3_1_5B_BASE_URL = `${BASE_URL}-bielik-v3.0/${NEXT_VERSION_TAG}`; @@ -1295,6 +1463,11 @@ const GEMMA4_E2B_XNNPACK_8DA4W: LLMModel = { tokenizerPath: `${GEMMA4_BASE_URL}/e2b/tokenizer.json`, tokenizerConfigPath: `${GEMMA4_BASE_URL}/e2b/tokenizer_config.json`, }; +const GEMMA4_E2B_VULKAN_8DA4W: LLMModel = { + modelPath: `${GEMMA4_BASE_URL}/e2b/vulkan/gemma_4_e2b_vulkan_8da4w.pte`, + tokenizerPath: `${GEMMA4_BASE_URL}/e2b/tokenizer.json`, + tokenizerConfigPath: `${GEMMA4_BASE_URL}/e2b/tokenizer_config.json`, +}; const GEMMA4_E2B_MLX_INT4: LLMModel = { modelPath: `${GEMMA4_BASE_URL}/e2b/mlx/gemma4_e2b_mlx_int4.pte`, tokenizerPath: `${GEMMA4_BASE_URL}/e2b/tokenizer.json`, @@ -1635,12 +1808,15 @@ export const models = { YOLO26_POSE: family({ SIZE_384: variants({ XNNPACK_FP32: YOLO26_POSE_384_XNNPACK_FP32, + COREML_FP16: YOLO26_POSE_384_COREML_FP16, }), SIZE_512: variants({ XNNPACK_FP32: YOLO26_POSE_512_XNNPACK_FP32, + COREML_FP16: YOLO26_POSE_512_COREML_FP16, }), SIZE_640: variants({ XNNPACK_FP32: YOLO26_POSE_640_XNNPACK_FP32, + COREML_FP16: YOLO26_POSE_640_COREML_FP16, }), }), /** @@ -1725,12 +1901,15 @@ export const models = { NANO: family({ SIZE_384: variants({ XNNPACK_FP32: YOLO26_NANO_SEG_384_XNNPACK_FP32, + COREML_FP16: YOLO26_NANO_SEG_384_COREML_FP16, }), SIZE_512: variants({ XNNPACK_FP32: YOLO26_NANO_SEG_512_XNNPACK_FP32, + COREML_FP16: YOLO26_NANO_SEG_512_COREML_FP16, }), SIZE_640: variants({ XNNPACK_FP32: YOLO26_NANO_SEG_640_XNNPACK_FP32, + COREML_FP16: YOLO26_NANO_SEG_640_COREML_FP16, }), }), /** @@ -1740,12 +1919,15 @@ export const models = { SMALL: family({ SIZE_384: variants({ XNNPACK_FP32: YOLO26_SMALL_SEG_384_XNNPACK_FP32, + COREML_FP16: YOLO26_SMALL_SEG_384_COREML_FP16, }), SIZE_512: variants({ XNNPACK_FP32: YOLO26_SMALL_SEG_512_XNNPACK_FP32, + COREML_FP16: YOLO26_SMALL_SEG_512_COREML_FP16, }), SIZE_640: variants({ XNNPACK_FP32: YOLO26_SMALL_SEG_640_XNNPACK_FP32, + COREML_FP16: YOLO26_SMALL_SEG_640_COREML_FP16, }), }), /** @@ -1755,12 +1937,15 @@ export const models = { MEDIUM: family({ SIZE_384: variants({ XNNPACK_FP32: YOLO26_MEDIUM_SEG_384_XNNPACK_FP32, + COREML_FP16: YOLO26_MEDIUM_SEG_384_COREML_FP16, }), SIZE_512: variants({ XNNPACK_FP32: YOLO26_MEDIUM_SEG_512_XNNPACK_FP32, + COREML_FP16: YOLO26_MEDIUM_SEG_512_COREML_FP16, }), SIZE_640: variants({ XNNPACK_FP32: YOLO26_MEDIUM_SEG_640_XNNPACK_FP32, + COREML_FP16: YOLO26_MEDIUM_SEG_640_COREML_FP16, }), }), /** @@ -1770,12 +1955,15 @@ export const models = { LARGE: family({ SIZE_384: variants({ XNNPACK_FP32: YOLO26_LARGE_SEG_384_XNNPACK_FP32, + COREML_FP16: YOLO26_LARGE_SEG_384_COREML_FP16, }), SIZE_512: variants({ XNNPACK_FP32: YOLO26_LARGE_SEG_512_XNNPACK_FP32, + COREML_FP16: YOLO26_LARGE_SEG_512_COREML_FP16, }), SIZE_640: variants({ XNNPACK_FP32: YOLO26_LARGE_SEG_640_XNNPACK_FP32, + COREML_FP16: YOLO26_LARGE_SEG_640_COREML_FP16, }), }), /** @@ -1785,12 +1973,15 @@ export const models = { XLARGE: family({ SIZE_384: variants({ XNNPACK_FP32: YOLO26_XLARGE_SEG_384_XNNPACK_FP32, + COREML_FP16: YOLO26_XLARGE_SEG_384_COREML_FP16, }), SIZE_512: variants({ XNNPACK_FP32: YOLO26_XLARGE_SEG_512_XNNPACK_FP32, + COREML_FP16: YOLO26_XLARGE_SEG_512_COREML_FP16, }), SIZE_640: variants({ XNNPACK_FP32: YOLO26_XLARGE_SEG_640_XNNPACK_FP32, + COREML_FP16: YOLO26_XLARGE_SEG_640_COREML_FP16, }), }), }), @@ -1880,6 +2071,7 @@ export const models = { TINY: variants( { XNNPACK_FP32: WHISPER_TINY_EN_XNNPACK_FP32, + XNNPACK_INT8: WHISPER_TINY_EN_XNNPACK_INT8, COREML_FP16: WHISPER_TINY_EN_COREML_FP16, MLX_BF16: WHISPER_TINY_EN_MLX_BF16, MLX_INT8: WHISPER_TINY_EN_MLX_INT8, @@ -1894,6 +2086,7 @@ export const models = { BASE: variants( { XNNPACK_FP32: WHISPER_BASE_EN_XNNPACK_FP32, + XNNPACK_INT8: WHISPER_BASE_EN_XNNPACK_INT8, COREML_FP16: WHISPER_BASE_EN_COREML_FP16, MLX_BF16: WHISPER_BASE_EN_MLX_BF16, MLX_INT8: WHISPER_BASE_EN_MLX_INT8, @@ -1908,6 +2101,7 @@ export const models = { SMALL: variants( { XNNPACK_FP32: WHISPER_SMALL_EN_XNNPACK_FP32, + XNNPACK_INT8: WHISPER_SMALL_EN_XNNPACK_INT8, COREML_FP16: WHISPER_SMALL_EN_COREML_FP16, MLX_INT8: WHISPER_SMALL_EN_MLX_INT8, }, @@ -1974,6 +2168,8 @@ export const models = { LFM2_5_VL_1_6B: variants({ XNNPACK_8DA4W: LFM2_5_VL_1_6B_XNNPACK_8DA4W, VULKAN_8DA4W: LFM2_5_VL_1_6B_VULKAN_8DA4W, + MLX_INT4: LFM2_5_VL_1_6B_MLX_INT4, + MLX_INT8: LFM2_5_VL_1_6B_MLX_INT8, }), /** * Bielik v3 1.5B bilingual Polish & English language model, developed by @@ -2140,6 +2336,7 @@ export const models = { */ GEMMA4_E2B: variants({ XNNPACK_8DA4W: GEMMA4_E2B_XNNPACK_8DA4W, + VULKAN_8DA4W: GEMMA4_E2B_VULKAN_8DA4W, MLX_INT4: GEMMA4_E2B_MLX_INT4, }), }, @@ -2156,6 +2353,7 @@ export const models = { */ ALL_MINILM_L6_V2: variants({ XNNPACK_FP32: ALL_MINILM_L6_V2_EMBEDDINGS, + COREML_FP16: ALL_MINILM_L6_V2_COREML_FP16, }), /** * High-quality 768-dimensional sentence transformer model based on MPNet. @@ -2170,6 +2368,7 @@ export const models = { */ MULTI_QA_MINILM_L6_COS_V1: variants({ XNNPACK_FP32: MULTI_QA_MINILM_L6_COS_V1_EMBEDDINGS, + COREML_FP16: MULTI_QA_MINILM_L6_COS_V1_COREML_FP16, }), /** * 768-dimensional sentence transformer fine-tuned specifically for @@ -2184,15 +2383,29 @@ export const models = { */ PARAPHRASE_MULTILINGUAL_MINILM_L12_V2: variants({ XNNPACK_8DA4W: PARAPHRASE_MULTILINGUAL_MINILM_L12_V2_EMBEDDINGS, + XNNPACK_FP32: PARAPHRASE_MULTILINGUAL_MINILM_L12_V2_XNNPACK_FP32, + COREML_FP16: PARAPHRASE_MULTILINGUAL_MINILM_L12_V2_COREML_FP16, }), /** * Multilingual sentence transformer supporting 50+ languages, based on * distilled Universal Sentence Encoder (512-dim output). */ - DISTILUSE_BASE_MULTILINGUAL_CASED_V2: variants({ - XNNPACK_8DA4W: DISTILUSE_BASE_MULTILINGUAL_CASED_V2_EMBEDDINGS, - MLX_INT8: DISTILUSE_BASE_MULTILINGUAL_CASED_V2_MLX_INT8, - }), + DISTILUSE_BASE_MULTILINGUAL_CASED_V2: variants( + { + XNNPACK_8DA4W: DISTILUSE_BASE_MULTILINGUAL_CASED_V2_EMBEDDINGS, + XNNPACK_FP32: DISTILUSE_BASE_MULTILINGUAL_CASED_V2_XNNPACK_FP32, + COREML_FP16: DISTILUSE_BASE_MULTILINGUAL_CASED_V2_COREML_FP16, + MLX_INT8: DISTILUSE_BASE_MULTILINGUAL_CASED_V2_MLX_INT8, + }, + // The one Core ML/MLX pair that does not go Core ML's way. On an + // iPhone 16, warm, over 150 embeds of five sentences in five languages: + // MLX int8 3.46 ms, Core ML fp16 3.96 ms, XNNPACK 8da4w 5.86 ms, stable + // when the arms are run in reverse. MLX is also half the download + // (140 MB against 271 MB) and skips Core ML's 786 ms first-use compile. + // Fidelity does not break the tie: against the XNNPACK reference both + // sit at 0.973 worst-case cosine. + { ios: 'MLX_INT8' } + ), /** * CLIP text encoder (ViT-B/32) mapping text queries into a 512-dimensional * joint text-image embedding space. Used in combination with @@ -2201,6 +2414,7 @@ export const models = { */ CLIP_VIT_BASE_PATCH32_TEXT: variants({ XNNPACK_FP32: CLIP_VIT_BASE_PATCH32_TEXT_EMBEDDINGS, + COREML_FP16: CLIP_VIT_BASE_PATCH32_TEXT_COREML_FP16, }), /** * Liquid AI LFM 2.5 350M parameter embedding model for asymmetric search @@ -2292,34 +2506,50 @@ export const models = { * Kokoro — a lightweight phoneme-driven Text-to-Speech model. Each language * entry bundles the matching model weights, grapheme-to-phoneme assets and * the voices available for that language, nested per backend. + * + * The Core ML builds are the registry's only fp32 Core ML exports — that is + * what is published, and fp32 keeps them off the Neural Engine, which is + * fp16-only. They still default on iOS because the duration predictor runs + * 14-21x faster warm there than the XNNPACK one on an iPhone 16. The cost + * is a one-time per-method Core ML compile on first use (~13s), cached + * across launches; reach for `XNNPACK_FP32` explicitly to avoid it. */ KOKORO: { EN_US: variants({ XNNPACK_FP32: KOKORO_EN_US_XNNPACK_FP32, + COREML_FP32: KOKORO_EN_US_COREML_FP32, }), EN_GB: variants({ XNNPACK_FP32: KOKORO_EN_GB_XNNPACK_FP32, + COREML_FP32: KOKORO_EN_GB_COREML_FP32, }), ES: variants({ XNNPACK_FP32: KOKORO_ES_XNNPACK_FP32, + COREML_FP32: KOKORO_ES_COREML_FP32, }), FR: variants({ XNNPACK_FP32: KOKORO_FR_XNNPACK_FP32, + COREML_FP32: KOKORO_FR_COREML_FP32, }), IT: variants({ XNNPACK_FP32: KOKORO_IT_XNNPACK_FP32, + COREML_FP32: KOKORO_IT_COREML_FP32, }), PT: variants({ XNNPACK_FP32: KOKORO_PT_XNNPACK_FP32, + COREML_FP32: KOKORO_PT_COREML_FP32, }), HI: variants({ XNNPACK_FP32: KOKORO_HI_XNNPACK_FP32, + COREML_FP32: KOKORO_HI_COREML_FP32, }), PL: variants({ XNNPACK_FP32: KOKORO_PL_XNNPACK_FP32, + COREML_FP32: KOKORO_PL_COREML_FP32, }), DE: variants({ XNNPACK_FP32: KOKORO_DE_XNNPACK_FP32, + COREML_FP32: KOKORO_DE_COREML_FP32, }), }, }, @@ -2341,6 +2571,7 @@ export const models = { */ PPOCRV6_SMALL: variants({ XNNPACK: PPOCRV6_SMALL_XNNPACK_INT8, + XNNPACK_FP32: PPOCRV6_SMALL_XNNPACK_FP32, COREML: PPOCRV6_SMALL_COREML_INT8, VULKAN: PPOCRV6_SMALL_VULKAN_FP16, }), From 96253f0be89f92eb4eebc6300a2053b5d8fe0e1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20S=C5=82uszniak?= Date: Sun, 30 Aug 2026 12:10:58 +0200 Subject: [PATCH 12/24] feat(models): wire the all-MiniLM-L6-v2 Vulkan export Android GPU build of the registry's most downloaded model. Under the platform rule it becomes the Android default, which is what the benchmark supports: on a Mali-G76 (Galaxy S10+) at the published 254 token shape it runs 169.8 ms against the XNNPACK build's 408.4 ms median, a 2.4x speedup, bit-identical across 10 executions and cosine 0.9999924 to the CPU reference. textEmbeddings gains vulkan in the download-libs feature map, without which DEFAULT would resolve to a backend the binary was never linked with. The sync test catches exactly this. --- packages/react-native-executorch/scripts/download-libs.js | 7 ++++--- packages/react-native-executorch/src/models.ts | 5 +++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/packages/react-native-executorch/scripts/download-libs.js b/packages/react-native-executorch/scripts/download-libs.js index e5f9077ae9..386d96f641 100644 --- a/packages/react-native-executorch/scripts/download-libs.js +++ b/packages/react-native-executorch/scripts/download-libs.js @@ -135,9 +135,10 @@ const FEATURE_MAP = { textToSpeech: { backends: ['xnnpack', 'coreml', 'mlx'], libs: ['phonemis'] }, // FSMN VAD — xnnpack only. vad: { backends: ['xnnpack'], libs: [] }, - // The MiniLM/CLIP-text/distiluse family ships coreml alongside xnnpack, and - // LFM2.5-Embedding and distiluse add MLX iOS exports. - textEmbeddings: { backends: ['xnnpack', 'coreml', 'mlx'], libs: [] }, + // The MiniLM/CLIP-text/distiluse family ships coreml alongside xnnpack, + // LFM2.5-Embedding and distiluse add MLX iOS exports, and all-MiniLM-L6-v2 + // adds a vulkan Android export. + textEmbeddings: { backends: ['xnnpack', 'coreml', 'mlx', 'vulkan'], libs: [] }, // CLIP's vision encoder ships xnnpack, coreml and mlx. imageEmbeddings: { backends: ['xnnpack', 'coreml', 'mlx'], libs: ['opencv'] }, // EfficientNet ships xnnpack + coreml. diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts index cf26227507..e7d5dc6c47 100644 --- a/packages/react-native-executorch/src/models.ts +++ b/packages/react-native-executorch/src/models.ts @@ -707,6 +707,10 @@ const ALL_MINILM_L6_V2_COREML_FP16: TextEmbedderModel = { modelPath: `${BASE_URL}-all-MiniLM-L6-v2/${NEXT_VERSION_TAG}/coreml/all_minilm_l6_v2_coreml_fp16.pte`, tokenizerPath: `${BASE_URL}-all-MiniLM-L6-v2/${NEXT_VERSION_TAG}/tokenizer.json`, }; +const ALL_MINILM_L6_V2_VULKAN_FP32: TextEmbedderModel = { + modelPath: `${BASE_URL}-all-MiniLM-L6-v2/${NEXT_VERSION_TAG}/vulkan/all_minilm_l6_v2_vulkan_fp32.pte`, + tokenizerPath: `${BASE_URL}-all-MiniLM-L6-v2/${NEXT_VERSION_TAG}/tokenizer.json`, +}; const ALL_MPNET_BASE_V2_EMBEDDINGS: TextEmbedderModel = { modelPath: `${BASE_URL}-all-mpnet-base-v2/${NEXT_VERSION_TAG}/xnnpack/all_mpnet_base_v2_xnnpack_fp32.pte`, tokenizerPath: `${BASE_URL}-all-mpnet-base-v2/${NEXT_VERSION_TAG}/tokenizer.json`, @@ -2354,6 +2358,7 @@ export const models = { ALL_MINILM_L6_V2: variants({ XNNPACK_FP32: ALL_MINILM_L6_V2_EMBEDDINGS, COREML_FP16: ALL_MINILM_L6_V2_COREML_FP16, + VULKAN_FP32: ALL_MINILM_L6_V2_VULKAN_FP32, }), /** * High-quality 768-dimensional sentence transformer model based on MPNet. From a4f78e14bc34d036351b4363eb0bf218c93fce40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20S=C5=82uszniak?= Date: Mon, 31 Aug 2026 13:43:17 +0200 Subject: [PATCH 13/24] feat(models): wire the whisper-tiny Vulkan exports --- apps/speech/app/audio-file-transcription/index.tsx | 10 ++++++++++ apps/speech/app/microphone-transcription/index.tsx | 10 ++++++++++ .../scripts/download-libs.js | 4 ++-- packages/react-native-executorch/src/models.ts | 14 ++++++++++++++ 4 files changed, 36 insertions(+), 2 deletions(-) diff --git a/apps/speech/app/audio-file-transcription/index.tsx b/apps/speech/app/audio-file-transcription/index.tsx index 1b58375c40..f6935f02b1 100644 --- a/apps/speech/app/audio-file-transcription/index.tsx +++ b/apps/speech/app/audio-file-transcription/index.tsx @@ -73,6 +73,16 @@ const MODELS = [ config: models.speechToText.WHISPER.TINY.MLX_BF16, disabled: Platform.OS !== 'ios', }, + { + name: 'Tiny Multilingual (Vulkan fp16)', + config: models.speechToText.WHISPER.TINY.VULKAN_FP16, + disabled: Platform.OS !== 'android', + }, + { + name: 'Tiny Multilingual (Vulkan int8)', + config: models.speechToText.WHISPER.TINY.VULKAN_INT8, + disabled: Platform.OS !== 'android', + }, { name: 'Base Multilingual (CPU)', config: models.speechToText.WHISPER.BASE.XNNPACK_FP32, diff --git a/apps/speech/app/microphone-transcription/index.tsx b/apps/speech/app/microphone-transcription/index.tsx index 1355e5991a..70ee8606de 100644 --- a/apps/speech/app/microphone-transcription/index.tsx +++ b/apps/speech/app/microphone-transcription/index.tsx @@ -66,6 +66,16 @@ const MODELS = [ config: models.speechToText.WHISPER.TINY.MLX_BF16, disabled: Platform.OS !== 'ios', }, + { + name: 'Tiny Multilingual (Vulkan fp16)', + config: models.speechToText.WHISPER.TINY.VULKAN_FP16, + disabled: Platform.OS !== 'android', + }, + { + name: 'Tiny Multilingual (Vulkan int8)', + config: models.speechToText.WHISPER.TINY.VULKAN_INT8, + disabled: Platform.OS !== 'android', + }, { name: 'Base Multilingual (CPU)', config: models.speechToText.WHISPER.BASE.XNNPACK_FP32, diff --git a/packages/react-native-executorch/scripts/download-libs.js b/packages/react-native-executorch/scripts/download-libs.js index 386d96f641..03f23eee68 100644 --- a/packages/react-native-executorch/scripts/download-libs.js +++ b/packages/react-native-executorch/scripts/download-libs.js @@ -129,8 +129,8 @@ const FEATURE_MAP = { multimodalLLM: { backends: ['xnnpack', 'mlx', 'vulkan'], libs: ['opencv'] }, // Privacy filter classifiers ship xnnpack + an MLX iOS export. privacyFilter: { backends: ['xnnpack', 'mlx'], libs: [] }, - // Whisper ships xnnpack, coreml and an MLX iOS export. - speechToText: { backends: ['xnnpack', 'coreml', 'mlx'], libs: [] }, + // Whisper ships xnnpack, coreml, an MLX iOS export and a Vulkan Android one. + speechToText: { backends: ['xnnpack', 'coreml', 'mlx', 'vulkan'], libs: [] }, // Kokoro ships xnnpack + coreml; Supertonic adds an MLX iOS export. textToSpeech: { backends: ['xnnpack', 'coreml', 'mlx'], libs: ['phonemis'] }, // FSMN VAD — xnnpack only. diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts index e7d5dc6c47..838a0d480f 100644 --- a/packages/react-native-executorch/src/models.ts +++ b/packages/react-native-executorch/src/models.ts @@ -867,6 +867,18 @@ const WHISPER_TINY_MLX_INT8: WhisperSttModel = { supportedLanguages: WHISPER_LANGUAGES, vadModel: FSMN_VAD_XNNPACK_FP32, }; +const WHISPER_TINY_VULKAN_FP16: WhisperSttModel = { + modelPath: `${BASE_URL}-whisper-tiny/${NEXT_VERSION_TAG}/vulkan/whisper_tiny_vulkan_fp16.pte`, + tokenizerPath: `${BASE_URL}-whisper-tiny/${NEXT_VERSION_TAG}/tokenizer.json`, + supportedLanguages: WHISPER_LANGUAGES, + vadModel: FSMN_VAD_XNNPACK_FP32, +}; +const WHISPER_TINY_VULKAN_INT8: WhisperSttModel = { + modelPath: `${BASE_URL}-whisper-tiny/${NEXT_VERSION_TAG}/vulkan/whisper_tiny_vulkan_int8.pte`, + tokenizerPath: `${BASE_URL}-whisper-tiny/${NEXT_VERSION_TAG}/tokenizer.json`, + supportedLanguages: WHISPER_LANGUAGES, + vadModel: FSMN_VAD_XNNPACK_FP32, +}; const WHISPER_BASE_EN_XNNPACK_FP32: WhisperSttModel<'en'> = { modelPath: `${BASE_URL}-whisper-base.en/${NEXT_VERSION_TAG}/xnnpack/whisper_base_en_xnnpack_fp32.pte`, @@ -2036,6 +2048,8 @@ export const models = { COREML_FP16: WHISPER_TINY_COREML_FP16, MLX_BF16: WHISPER_TINY_MLX_BF16, MLX_INT8: WHISPER_TINY_MLX_INT8, + VULKAN_FP16: WHISPER_TINY_VULKAN_FP16, + VULKAN_INT8: WHISPER_TINY_VULKAN_INT8, }, // Core ML over MLX, see the note on WHISPER. { ios: 'COREML_FP16' } From 66ab3cd2672f09147ae697f2340ac8b6f349960e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20S=C5=82uszniak?= Date: Mon, 31 Aug 2026 16:49:38 +0200 Subject: [PATCH 14/24] feat(models): wire the remaining whisper Vulkan sizes tiny.en, base, base.en, small and small.en, matching the multilingual tiny entries already added. Encode on Vulkan is 1.63x to 1.94x the XNNPACK build across the six sizes on an Adreno 840, with transcripts identical to the fp32 reference, so each takes the Android default under the platform rule. The demo gains the three Vulkan entries that are useful to compare against their CPU counterparts; the rest are reachable by name. --- .../app/audio-file-transcription/index.tsx | 15 ++++ .../app/microphone-transcription/index.tsx | 15 ++++ .../react-native-executorch/src/models.ts | 70 +++++++++++++++++++ 3 files changed, 100 insertions(+) diff --git a/apps/speech/app/audio-file-transcription/index.tsx b/apps/speech/app/audio-file-transcription/index.tsx index f6935f02b1..1bba02bab5 100644 --- a/apps/speech/app/audio-file-transcription/index.tsx +++ b/apps/speech/app/audio-file-transcription/index.tsx @@ -83,6 +83,21 @@ const MODELS = [ config: models.speechToText.WHISPER.TINY.VULKAN_INT8, disabled: Platform.OS !== 'android', }, + { + name: 'Tiny English (Vulkan fp16)', + config: models.speechToText.WHISPER.EN.TINY.VULKAN_FP16, + disabled: Platform.OS !== 'android', + }, + { + name: 'Base Multilingual (Vulkan fp16)', + config: models.speechToText.WHISPER.BASE.VULKAN_FP16, + disabled: Platform.OS !== 'android', + }, + { + name: 'Small Multilingual (Vulkan fp16)', + config: models.speechToText.WHISPER.SMALL.VULKAN_FP16, + disabled: Platform.OS !== 'android', + }, { name: 'Base Multilingual (CPU)', config: models.speechToText.WHISPER.BASE.XNNPACK_FP32, diff --git a/apps/speech/app/microphone-transcription/index.tsx b/apps/speech/app/microphone-transcription/index.tsx index 70ee8606de..b30ec6b0b9 100644 --- a/apps/speech/app/microphone-transcription/index.tsx +++ b/apps/speech/app/microphone-transcription/index.tsx @@ -76,6 +76,21 @@ const MODELS = [ config: models.speechToText.WHISPER.TINY.VULKAN_INT8, disabled: Platform.OS !== 'android', }, + { + name: 'Tiny English (Vulkan fp16)', + config: models.speechToText.WHISPER.EN.TINY.VULKAN_FP16, + disabled: Platform.OS !== 'android', + }, + { + name: 'Base Multilingual (Vulkan fp16)', + config: models.speechToText.WHISPER.BASE.VULKAN_FP16, + disabled: Platform.OS !== 'android', + }, + { + name: 'Small Multilingual (Vulkan fp16)', + config: models.speechToText.WHISPER.SMALL.VULKAN_FP16, + disabled: Platform.OS !== 'android', + }, { name: 'Base Multilingual (CPU)', config: models.speechToText.WHISPER.BASE.XNNPACK_FP32, diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts index 838a0d480f..e0ad01310d 100644 --- a/packages/react-native-executorch/src/models.ts +++ b/packages/react-native-executorch/src/models.ts @@ -842,6 +842,18 @@ const WHISPER_TINY_EN_MLX_INT8: WhisperSttModel<'en'> = { supportedLanguages: ['en'], vadModel: FSMN_VAD_XNNPACK_FP32, }; +const WHISPER_TINY_EN_VULKAN_FP16: WhisperSttModel<'en'> = { + modelPath: `${BASE_URL}-whisper-tiny.en/${NEXT_VERSION_TAG}/vulkan/whisper_tiny_en_vulkan_fp16.pte`, + tokenizerPath: `${BASE_URL}-whisper-tiny.en/${NEXT_VERSION_TAG}/tokenizer.json`, + supportedLanguages: ['en'], + vadModel: FSMN_VAD_XNNPACK_FP32, +}; +const WHISPER_TINY_EN_VULKAN_INT8: WhisperSttModel<'en'> = { + modelPath: `${BASE_URL}-whisper-tiny.en/${NEXT_VERSION_TAG}/vulkan/whisper_tiny_en_vulkan_int8.pte`, + tokenizerPath: `${BASE_URL}-whisper-tiny.en/${NEXT_VERSION_TAG}/tokenizer.json`, + supportedLanguages: ['en'], + vadModel: FSMN_VAD_XNNPACK_FP32, +}; const WHISPER_TINY_XNNPACK_FP32: WhisperSttModel = { modelPath: `${BASE_URL}-whisper-tiny/${NEXT_VERSION_TAG}/xnnpack/whisper_tiny_xnnpack_fp32.pte`, @@ -910,6 +922,18 @@ const WHISPER_BASE_EN_MLX_INT8: WhisperSttModel<'en'> = { supportedLanguages: ['en'], vadModel: FSMN_VAD_XNNPACK_FP32, }; +const WHISPER_BASE_EN_VULKAN_FP16: WhisperSttModel<'en'> = { + modelPath: `${BASE_URL}-whisper-base.en/${NEXT_VERSION_TAG}/vulkan/whisper_base_en_vulkan_fp16.pte`, + tokenizerPath: `${BASE_URL}-whisper-base.en/${NEXT_VERSION_TAG}/tokenizer.json`, + supportedLanguages: ['en'], + vadModel: FSMN_VAD_XNNPACK_FP32, +}; +const WHISPER_BASE_EN_VULKAN_INT8: WhisperSttModel<'en'> = { + modelPath: `${BASE_URL}-whisper-base.en/${NEXT_VERSION_TAG}/vulkan/whisper_base_en_vulkan_int8.pte`, + tokenizerPath: `${BASE_URL}-whisper-base.en/${NEXT_VERSION_TAG}/tokenizer.json`, + supportedLanguages: ['en'], + vadModel: FSMN_VAD_XNNPACK_FP32, +}; const WHISPER_BASE_XNNPACK_FP32: WhisperSttModel = { modelPath: `${BASE_URL}-whisper-base/${NEXT_VERSION_TAG}/xnnpack/whisper_base_xnnpack_fp32.pte`, @@ -935,6 +959,18 @@ const WHISPER_BASE_MLX_INT8: WhisperSttModel = { supportedLanguages: WHISPER_LANGUAGES, vadModel: FSMN_VAD_XNNPACK_FP32, }; +const WHISPER_BASE_VULKAN_FP16: WhisperSttModel = { + modelPath: `${BASE_URL}-whisper-base/${NEXT_VERSION_TAG}/vulkan/whisper_base_vulkan_fp16.pte`, + tokenizerPath: `${BASE_URL}-whisper-base/${NEXT_VERSION_TAG}/tokenizer.json`, + supportedLanguages: WHISPER_LANGUAGES, + vadModel: FSMN_VAD_XNNPACK_FP32, +}; +const WHISPER_BASE_VULKAN_INT8: WhisperSttModel = { + modelPath: `${BASE_URL}-whisper-base/${NEXT_VERSION_TAG}/vulkan/whisper_base_vulkan_int8.pte`, + tokenizerPath: `${BASE_URL}-whisper-base/${NEXT_VERSION_TAG}/tokenizer.json`, + supportedLanguages: WHISPER_LANGUAGES, + vadModel: FSMN_VAD_XNNPACK_FP32, +}; const WHISPER_SMALL_EN_XNNPACK_FP32: WhisperSttModel<'en'> = { modelPath: `${BASE_URL}-whisper-small.en/${NEXT_VERSION_TAG}/xnnpack/whisper_small_en_xnnpack_fp32.pte`, @@ -960,6 +996,18 @@ const WHISPER_SMALL_EN_MLX_INT8: WhisperSttModel<'en'> = { supportedLanguages: ['en'], vadModel: FSMN_VAD_XNNPACK_FP32, }; +const WHISPER_SMALL_EN_VULKAN_FP16: WhisperSttModel<'en'> = { + modelPath: `${BASE_URL}-whisper-small.en/${NEXT_VERSION_TAG}/vulkan/whisper_small_en_vulkan_fp16.pte`, + tokenizerPath: `${BASE_URL}-whisper-small.en/${NEXT_VERSION_TAG}/tokenizer.json`, + supportedLanguages: ['en'], + vadModel: FSMN_VAD_XNNPACK_FP32, +}; +const WHISPER_SMALL_EN_VULKAN_INT8: WhisperSttModel<'en'> = { + modelPath: `${BASE_URL}-whisper-small.en/${NEXT_VERSION_TAG}/vulkan/whisper_small_en_vulkan_int8.pte`, + tokenizerPath: `${BASE_URL}-whisper-small.en/${NEXT_VERSION_TAG}/tokenizer.json`, + supportedLanguages: ['en'], + vadModel: FSMN_VAD_XNNPACK_FP32, +}; const WHISPER_SMALL_XNNPACK_FP32: WhisperSttModel = { modelPath: `${BASE_URL}-whisper-small/${NEXT_VERSION_TAG}/xnnpack/whisper_small_xnnpack_fp32.pte`, @@ -979,6 +1027,18 @@ const WHISPER_SMALL_MLX_INT8: WhisperSttModel = { supportedLanguages: WHISPER_LANGUAGES, vadModel: FSMN_VAD_XNNPACK_FP32, }; +const WHISPER_SMALL_VULKAN_FP16: WhisperSttModel = { + modelPath: `${BASE_URL}-whisper-small/${NEXT_VERSION_TAG}/vulkan/whisper_small_vulkan_fp16.pte`, + tokenizerPath: `${BASE_URL}-whisper-small/${NEXT_VERSION_TAG}/tokenizer.json`, + supportedLanguages: WHISPER_LANGUAGES, + vadModel: FSMN_VAD_XNNPACK_FP32, +}; +const WHISPER_SMALL_VULKAN_INT8: WhisperSttModel = { + modelPath: `${BASE_URL}-whisper-small/${NEXT_VERSION_TAG}/vulkan/whisper_small_vulkan_int8.pte`, + tokenizerPath: `${BASE_URL}-whisper-small/${NEXT_VERSION_TAG}/tokenizer.json`, + supportedLanguages: WHISPER_LANGUAGES, + vadModel: FSMN_VAD_XNNPACK_FP32, +}; // ============================================================================= // Text to Image @@ -2064,6 +2124,8 @@ export const models = { COREML_FP16: WHISPER_BASE_COREML_FP16, MLX_BF16: WHISPER_BASE_MLX_BF16, MLX_INT8: WHISPER_BASE_MLX_INT8, + VULKAN_FP16: WHISPER_BASE_VULKAN_FP16, + VULKAN_INT8: WHISPER_BASE_VULKAN_INT8, }, // Core ML over MLX, see the note on WHISPER. { ios: 'COREML_FP16' } @@ -2077,6 +2139,8 @@ export const models = { XNNPACK_FP32: WHISPER_SMALL_XNNPACK_FP32, COREML_FP16: WHISPER_SMALL_COREML_FP16, MLX_INT8: WHISPER_SMALL_MLX_INT8, + VULKAN_FP16: WHISPER_SMALL_VULKAN_FP16, + VULKAN_INT8: WHISPER_SMALL_VULKAN_INT8, }, // Core ML over MLX, see the note on WHISPER. { ios: 'COREML_FP16' } @@ -2093,6 +2157,8 @@ export const models = { COREML_FP16: WHISPER_TINY_EN_COREML_FP16, MLX_BF16: WHISPER_TINY_EN_MLX_BF16, MLX_INT8: WHISPER_TINY_EN_MLX_INT8, + VULKAN_FP16: WHISPER_TINY_EN_VULKAN_FP16, + VULKAN_INT8: WHISPER_TINY_EN_VULKAN_INT8, }, // Core ML over MLX, see the note on WHISPER. { ios: 'COREML_FP16' } @@ -2108,6 +2174,8 @@ export const models = { COREML_FP16: WHISPER_BASE_EN_COREML_FP16, MLX_BF16: WHISPER_BASE_EN_MLX_BF16, MLX_INT8: WHISPER_BASE_EN_MLX_INT8, + VULKAN_FP16: WHISPER_BASE_EN_VULKAN_FP16, + VULKAN_INT8: WHISPER_BASE_EN_VULKAN_INT8, }, // Core ML over MLX, see the note on WHISPER. { ios: 'COREML_FP16' } @@ -2122,6 +2190,8 @@ export const models = { XNNPACK_INT8: WHISPER_SMALL_EN_XNNPACK_INT8, COREML_FP16: WHISPER_SMALL_EN_COREML_FP16, MLX_INT8: WHISPER_SMALL_EN_MLX_INT8, + VULKAN_FP16: WHISPER_SMALL_EN_VULKAN_FP16, + VULKAN_INT8: WHISPER_SMALL_EN_VULKAN_INT8, }, // Core ML over MLX, see the note on WHISPER. { ios: 'COREML_FP16' } From b785ea8b0e3d7b3f224f736d971a8b53578f01e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20S=C5=82uszniak?= Date: Mon, 31 Aug 2026 20:14:01 +0200 Subject: [PATCH 15/24] feat(models): default the embedding models to Vulkan fp16 on Android Adds Vulkan fp16 variants for CLIP ViT-B/32 vision and text and for all-mpnet-base-v2, and moves all-MiniLM-L6-v2 from fp32 to fp16. Vulkan leads BACKEND_ORDER.android, so each of these becomes the Android default. Measured against XNNPACK, medians of 5 interleaved rounds of 50 iterations, on an S26 Ultra (Adreno 840) and an S10+ (Mali-G76): Adreno Mali MiniLM 2.05x 2.85x CLIP vision tie 2.06x CLIP text 1.10x 2.00x mpnet 1.35x 1.90x fp16 rather than fp32 throughout. fp32 is the faster arm on Mali, but it loses to XNNPACK on Adreno for CLIP text and mpnet, so fp16 is the only precision that wins on both parts. It also halves every download: MiniLM 90 to 45 MB, CLIP 352 to 176 and 254 to 127, mpnet 435 to 218. CLIP vision is a tie on Adreno rather than a win. It is defaulted anyway: it costs that device nothing, doubles throughput on the older part, and halves the download everywhere. imageEmbeddings now provisions vulkan in download-libs. Without it the backend is never downloaded and the model silently falls back to XNNPACK. Outputs match the CPU reference at cosine >= 0.99995 on both GPUs, and both CLIP encoders are deterministic over 60 consecutive runs on each. --- .../scripts/download-libs.js | 9 ++++---- .../react-native-executorch/src/models.ts | 21 ++++++++++++++++--- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/packages/react-native-executorch/scripts/download-libs.js b/packages/react-native-executorch/scripts/download-libs.js index 03f23eee68..56f9658b9b 100644 --- a/packages/react-native-executorch/scripts/download-libs.js +++ b/packages/react-native-executorch/scripts/download-libs.js @@ -136,11 +136,12 @@ const FEATURE_MAP = { // FSMN VAD — xnnpack only. vad: { backends: ['xnnpack'], libs: [] }, // The MiniLM/CLIP-text/distiluse family ships coreml alongside xnnpack, - // LFM2.5-Embedding and distiluse add MLX iOS exports, and all-MiniLM-L6-v2 - // adds a vulkan Android export. + // LFM2.5-Embedding and distiluse add MLX iOS exports, and all-MiniLM-L6-v2, + // CLIP-text and all-mpnet-base-v2 add vulkan Android exports. textEmbeddings: { backends: ['xnnpack', 'coreml', 'mlx', 'vulkan'], libs: [] }, - // CLIP's vision encoder ships xnnpack, coreml and mlx. - imageEmbeddings: { backends: ['xnnpack', 'coreml', 'mlx'], libs: ['opencv'] }, + // CLIP's vision encoder ships xnnpack, coreml and mlx, plus a vulkan Android + // export. + imageEmbeddings: { backends: ['xnnpack', 'coreml', 'mlx', 'vulkan'], libs: ['opencv'] }, // EfficientNet ships xnnpack + coreml. classification: { backends: ['xnnpack', 'coreml'], libs: ['opencv'] }, // YOLO is xnnpack-only, ssdlite/rf_detr add coreml → union. diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts index e0ad01310d..372adb740b 100644 --- a/packages/react-native-executorch/src/models.ts +++ b/packages/react-native-executorch/src/models.ts @@ -707,14 +707,18 @@ const ALL_MINILM_L6_V2_COREML_FP16: TextEmbedderModel = { modelPath: `${BASE_URL}-all-MiniLM-L6-v2/${NEXT_VERSION_TAG}/coreml/all_minilm_l6_v2_coreml_fp16.pte`, tokenizerPath: `${BASE_URL}-all-MiniLM-L6-v2/${NEXT_VERSION_TAG}/tokenizer.json`, }; -const ALL_MINILM_L6_V2_VULKAN_FP32: TextEmbedderModel = { - modelPath: `${BASE_URL}-all-MiniLM-L6-v2/${NEXT_VERSION_TAG}/vulkan/all_minilm_l6_v2_vulkan_fp32.pte`, +const ALL_MINILM_L6_V2_VULKAN_FP16: TextEmbedderModel = { + modelPath: `${BASE_URL}-all-MiniLM-L6-v2/${NEXT_VERSION_TAG}/vulkan/all_minilm_l6_v2_vulkan_fp16.pte`, tokenizerPath: `${BASE_URL}-all-MiniLM-L6-v2/${NEXT_VERSION_TAG}/tokenizer.json`, }; const ALL_MPNET_BASE_V2_EMBEDDINGS: TextEmbedderModel = { modelPath: `${BASE_URL}-all-mpnet-base-v2/${NEXT_VERSION_TAG}/xnnpack/all_mpnet_base_v2_xnnpack_fp32.pte`, tokenizerPath: `${BASE_URL}-all-mpnet-base-v2/${NEXT_VERSION_TAG}/tokenizer.json`, }; +const ALL_MPNET_BASE_V2_VULKAN_FP16: TextEmbedderModel = { + modelPath: `${BASE_URL}-all-mpnet-base-v2/${NEXT_VERSION_TAG}/vulkan/all_mpnet_base_v2_vulkan_fp16.pte`, + tokenizerPath: `${BASE_URL}-all-mpnet-base-v2/${NEXT_VERSION_TAG}/tokenizer.json`, +}; const MULTI_QA_MINILM_L6_COS_V1_EMBEDDINGS: TextEmbedderModel = { modelPath: `${BASE_URL}-multi-qa-MiniLM-L6-cos-v1/${NEXT_VERSION_TAG}/xnnpack/multi_qa_minilm_l6_cos_v1_xnnpack_fp32.pte`, tokenizerPath: `${BASE_URL}-multi-qa-MiniLM-L6-cos-v1/${NEXT_VERSION_TAG}/tokenizer.json`, @@ -763,6 +767,10 @@ const CLIP_VIT_BASE_PATCH32_TEXT_COREML_FP16: TextEmbedderModel = { modelPath: `${BASE_URL}-clip-vit-base-patch32/${NEXT_VERSION_TAG}/coreml/clip_vit_base_patch32_text_coreml_fp16.pte`, tokenizerPath: `${BASE_URL}-clip-vit-base-patch32/${NEXT_VERSION_TAG}/tokenizer.json`, }; +const CLIP_VIT_BASE_PATCH32_TEXT_VULKAN_FP16: TextEmbedderModel = { + modelPath: `${BASE_URL}-clip-vit-base-patch32/${NEXT_VERSION_TAG}/vulkan/clip_vit_base_patch32_text_vulkan_fp16.pte`, + tokenizerPath: `${BASE_URL}-clip-vit-base-patch32/${NEXT_VERSION_TAG}/tokenizer.json`, +}; const LFM2_5_EMBEDDING_350M_EMBEDDINGS: TextEmbedderModel = { modelPath: `${BASE_URL}-lfm2.5-embedding-350m/${NEXT_VERSION_TAG}/xnnpack/lfm_2_5_embedding_350m_xnnpack_8da4w.pte`, tokenizerPath: `${BASE_URL}-lfm2.5-embedding-350m/${NEXT_VERSION_TAG}/tokenizer.json`, @@ -794,6 +802,10 @@ const CLIP_VIT_BASE_PATCH32_IMAGE_MLX_INT8: ImageEmbedderModel = { modelPath: `${BASE_URL}-clip-vit-base-patch32/${NEXT_VERSION_TAG}/mlx/clip_vit_base_patch32_image_mlx_int8.pte`, modelOpts: CLIP_IMAGE_EMBEDDINGS_OPTS, }; +const CLIP_VIT_BASE_PATCH32_IMAGE_VULKAN_FP16: ImageEmbedderModel = { + modelPath: `${BASE_URL}-clip-vit-base-patch32/${NEXT_VERSION_TAG}/vulkan/clip_vit_base_patch32_image_vulkan_fp16.pte`, + modelOpts: CLIP_IMAGE_EMBEDDINGS_OPTS, +}; // ============================================================================= // Voice Activity Detection @@ -2442,7 +2454,7 @@ export const models = { ALL_MINILM_L6_V2: variants({ XNNPACK_FP32: ALL_MINILM_L6_V2_EMBEDDINGS, COREML_FP16: ALL_MINILM_L6_V2_COREML_FP16, - VULKAN_FP32: ALL_MINILM_L6_V2_VULKAN_FP32, + VULKAN_FP16: ALL_MINILM_L6_V2_VULKAN_FP16, }), /** * High-quality 768-dimensional sentence transformer model based on MPNet. @@ -2450,6 +2462,7 @@ export const models = { */ ALL_MPNET_BASE_V2: variants({ XNNPACK_FP32: ALL_MPNET_BASE_V2_EMBEDDINGS, + VULKAN_FP16: ALL_MPNET_BASE_V2_VULKAN_FP16, }), /** * 384-dimensional sentence transformer fine-tuned specifically for semantic @@ -2504,6 +2517,7 @@ export const models = { CLIP_VIT_BASE_PATCH32_TEXT: variants({ XNNPACK_FP32: CLIP_VIT_BASE_PATCH32_TEXT_EMBEDDINGS, COREML_FP16: CLIP_VIT_BASE_PATCH32_TEXT_COREML_FP16, + VULKAN_FP16: CLIP_VIT_BASE_PATCH32_TEXT_VULKAN_FP16, }), /** * Liquid AI LFM 2.5 350M parameter embedding model for asymmetric search @@ -2554,6 +2568,7 @@ export const models = { XNNPACK_FP32: CLIP_VIT_BASE_PATCH32_IMAGE_XNNPACK_FP32, COREML_FP16: CLIP_VIT_BASE_PATCH32_IMAGE_COREML_FP16, MLX_INT8: CLIP_VIT_BASE_PATCH32_IMAGE_MLX_INT8, + VULKAN_FP16: CLIP_VIT_BASE_PATCH32_IMAGE_VULKAN_FP16, }, // Core ML over MLX: 3.5 ms against 14.1 on an iPhone 16, the widest // margin of any pair that ships both. From 5916c4ffae4f91e99f5135d8c7830c6e04d55c25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20S=C5=82uszniak?= Date: Mon, 31 Aug 2026 22:31:46 +0200 Subject: [PATCH 16/24] feat(models): add Vulkan fp16 for the remaining sentence-transformer embedders Wires multi-qa-MiniLM-L6-cos-v1, multi-qa-mpnet-base-dot-v1 and paraphrase-multilingual-MiniLM-L12-v2. Vulkan leads BACKEND_ORDER.android, so each becomes the Android default. Against XNNPACK, medians of 5 interleaved rounds of 50 iterations, on an S26 Ultra (Adreno 840) and an S10+ (Mali-G76): Adreno Mali multi_qa_minilm 1.63x 2.45x paraphrase_ml 1.40x 1.50x multi_qa_mpnet 1.14x (noisy) 1.60x multi_qa_mpnet is the weakest of the three: both arms swing widely on Adreno (vk 123-306 ms against xnn 204-319), so that figure is indicative only. On Mali its worst sample still beats XNNPACK's best, so the direction holds. Each halves its download, paraphrase-multilingual most of all at 470 to 235 MB. All three verified against the CPU reference at cosine >= 0.99997 on both GPUs. textEmbeddings already provisions vulkan in download-libs, so no change there. --- packages/react-native-executorch/src/models.ts | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts index 372adb740b..04416af892 100644 --- a/packages/react-native-executorch/src/models.ts +++ b/packages/react-native-executorch/src/models.ts @@ -727,10 +727,18 @@ const MULTI_QA_MINILM_L6_COS_V1_COREML_FP16: TextEmbedderModel = { modelPath: `${BASE_URL}-multi-qa-MiniLM-L6-cos-v1/${NEXT_VERSION_TAG}/coreml/multi_qa_minilm_l6_cos_v1_coreml_fp16.pte`, tokenizerPath: `${BASE_URL}-multi-qa-MiniLM-L6-cos-v1/${NEXT_VERSION_TAG}/tokenizer.json`, }; +const MULTI_QA_MINILM_L6_COS_V1_VULKAN_FP16: TextEmbedderModel = { + modelPath: `${BASE_URL}-multi-qa-MiniLM-L6-cos-v1/${NEXT_VERSION_TAG}/vulkan/multi_qa_minilm_l6_cos_v1_vulkan_fp16.pte`, + tokenizerPath: `${BASE_URL}-multi-qa-MiniLM-L6-cos-v1/${NEXT_VERSION_TAG}/tokenizer.json`, +}; const MULTI_QA_MPNET_BASE_DOT_V1_EMBEDDINGS: TextEmbedderModel = { modelPath: `${BASE_URL}-multi-qa-mpnet-base-dot-v1/${NEXT_VERSION_TAG}/xnnpack/multi_qa_mpnet_base_dot_v1_xnnpack_fp32.pte`, tokenizerPath: `${BASE_URL}-multi-qa-mpnet-base-dot-v1/${NEXT_VERSION_TAG}/tokenizer.json`, }; +const MULTI_QA_MPNET_BASE_DOT_V1_VULKAN_FP16: TextEmbedderModel = { + modelPath: `${BASE_URL}-multi-qa-mpnet-base-dot-v1/${NEXT_VERSION_TAG}/vulkan/multi_qa_mpnet_base_dot_v1_vulkan_fp16.pte`, + tokenizerPath: `${BASE_URL}-multi-qa-mpnet-base-dot-v1/${NEXT_VERSION_TAG}/tokenizer.json`, +}; const PARAPHRASE_MULTILINGUAL_MINILM_L12_V2_EMBEDDINGS: TextEmbedderModel = { modelPath: `${BASE_URL}-paraphrase-multilingual-MiniLM-L12-v2/${NEXT_VERSION_TAG}/xnnpack/paraphrase_multilingual_minilm_l12_v2_xnnpack_8da4w.pte`, tokenizerPath: `${BASE_URL}-paraphrase-multilingual-MiniLM-L12-v2/${NEXT_VERSION_TAG}/tokenizer.json`, @@ -743,6 +751,10 @@ const PARAPHRASE_MULTILINGUAL_MINILM_L12_V2_COREML_FP16: TextEmbedderModel = { modelPath: `${BASE_URL}-paraphrase-multilingual-MiniLM-L12-v2/${NEXT_VERSION_TAG}/coreml/paraphrase_multilingual_minilm_l12_v2_coreml_fp16.pte`, tokenizerPath: `${BASE_URL}-paraphrase-multilingual-MiniLM-L12-v2/${NEXT_VERSION_TAG}/tokenizer.json`, }; +const PARAPHRASE_MULTILINGUAL_MINILM_L12_V2_VULKAN_FP16: TextEmbedderModel = { + modelPath: `${BASE_URL}-paraphrase-multilingual-MiniLM-L12-v2/${NEXT_VERSION_TAG}/vulkan/paraphrase_multilingual_minilm_l12_v2_vulkan_fp16.pte`, + tokenizerPath: `${BASE_URL}-paraphrase-multilingual-MiniLM-L12-v2/${NEXT_VERSION_TAG}/tokenizer.json`, +}; const DISTILUSE_BASE_MULTILINGUAL_CASED_V2_EMBEDDINGS: TextEmbedderModel = { modelPath: `${BASE_URL}-distiluse-base-multilingual-cased-v2/${NEXT_VERSION_TAG}/xnnpack/distiluse_base_multilingual_cased_v2_xnnpack_8da4w.pte`, tokenizerPath: `${BASE_URL}-distiluse-base-multilingual-cased-v2/${NEXT_VERSION_TAG}/tokenizer.json`, @@ -2471,6 +2483,7 @@ export const models = { MULTI_QA_MINILM_L6_COS_V1: variants({ XNNPACK_FP32: MULTI_QA_MINILM_L6_COS_V1_EMBEDDINGS, COREML_FP16: MULTI_QA_MINILM_L6_COS_V1_COREML_FP16, + VULKAN_FP16: MULTI_QA_MINILM_L6_COS_V1_VULKAN_FP16, }), /** * 768-dimensional sentence transformer fine-tuned specifically for @@ -2478,6 +2491,7 @@ export const models = { */ MULTI_QA_MPNET_BASE_DOT_V1: variants({ XNNPACK_FP32: MULTI_QA_MPNET_BASE_DOT_V1_EMBEDDINGS, + VULKAN_FP16: MULTI_QA_MPNET_BASE_DOT_V1_VULKAN_FP16, }), /** * 384-dimensional sentence transformer supporting 50+ languages for @@ -2487,6 +2501,7 @@ export const models = { XNNPACK_8DA4W: PARAPHRASE_MULTILINGUAL_MINILM_L12_V2_EMBEDDINGS, XNNPACK_FP32: PARAPHRASE_MULTILINGUAL_MINILM_L12_V2_XNNPACK_FP32, COREML_FP16: PARAPHRASE_MULTILINGUAL_MINILM_L12_V2_COREML_FP16, + VULKAN_FP16: PARAPHRASE_MULTILINGUAL_MINILM_L12_V2_VULKAN_FP16, }), /** * Multilingual sentence transformer supporting 50+ languages, based on From a3d87949a1afa6a7af938b075af2ace2fe081ab5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20S=C5=82uszniak?= Date: Tue, 1 Sep 2026 09:48:56 +0200 Subject: [PATCH 17/24] feat(models): add Vulkan fp16 for distiluse Completes the sentence-transformer family on Vulkan. distiluse is the one embedder whose XNNPACK Android default is quantized (8da4w) rather than fp32, so this is not the speedup the others were. Medians at 126 tokens, 5 interleaved rounds, against that 8da4w default: Adreno 840 13.72 vs 13.60 ms (a tie), Mali-G76 155.75 vs 190.77 ms (1.22x). What it does buy everywhere is accuracy and size. Cosine against the fp32 CPU reference goes 0.962244 -> 0.999995, and the download drops 393 -> 270 MB. 60/60 runs bit-identical on both GPUs. --- packages/react-native-executorch/scripts/download-libs.js | 4 ++-- packages/react-native-executorch/src/models.ts | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/react-native-executorch/scripts/download-libs.js b/packages/react-native-executorch/scripts/download-libs.js index 56f9658b9b..5c97a39cc1 100644 --- a/packages/react-native-executorch/scripts/download-libs.js +++ b/packages/react-native-executorch/scripts/download-libs.js @@ -136,8 +136,8 @@ const FEATURE_MAP = { // FSMN VAD — xnnpack only. vad: { backends: ['xnnpack'], libs: [] }, // The MiniLM/CLIP-text/distiluse family ships coreml alongside xnnpack, - // LFM2.5-Embedding and distiluse add MLX iOS exports, and all-MiniLM-L6-v2, - // CLIP-text and all-mpnet-base-v2 add vulkan Android exports. + // LFM2.5-Embedding and distiluse add MLX iOS exports, and every sentence + // transformer now ships a vulkan Android export. textEmbeddings: { backends: ['xnnpack', 'coreml', 'mlx', 'vulkan'], libs: [] }, // CLIP's vision encoder ships xnnpack, coreml and mlx, plus a vulkan Android // export. diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts index 04416af892..3e75228c90 100644 --- a/packages/react-native-executorch/src/models.ts +++ b/packages/react-native-executorch/src/models.ts @@ -771,6 +771,10 @@ const DISTILUSE_BASE_MULTILINGUAL_CASED_V2_MLX_INT8: TextEmbedderModel = { modelPath: `${BASE_URL}-distiluse-base-multilingual-cased-v2/${NEXT_VERSION_TAG}/mlx/distiluse_base_multilingual_cased_v2_mlx_int8.pte`, tokenizerPath: `${BASE_URL}-distiluse-base-multilingual-cased-v2/${NEXT_VERSION_TAG}/tokenizer.json`, }; +const DISTILUSE_BASE_MULTILINGUAL_CASED_V2_VULKAN_FP16: TextEmbedderModel = { + modelPath: `${BASE_URL}-distiluse-base-multilingual-cased-v2/${NEXT_VERSION_TAG}/vulkan/distiluse_base_multilingual_cased_v2_vulkan_fp16.pte`, + tokenizerPath: `${BASE_URL}-distiluse-base-multilingual-cased-v2/${NEXT_VERSION_TAG}/tokenizer.json`, +}; const CLIP_VIT_BASE_PATCH32_TEXT_EMBEDDINGS: TextEmbedderModel = { modelPath: `${BASE_URL}-clip-vit-base-patch32/${NEXT_VERSION_TAG}/xnnpack/clip_vit_base_patch32_text_xnnpack_fp32.pte`, tokenizerPath: `${BASE_URL}-clip-vit-base-patch32/${NEXT_VERSION_TAG}/tokenizer.json`, @@ -2513,6 +2517,7 @@ export const models = { XNNPACK_FP32: DISTILUSE_BASE_MULTILINGUAL_CASED_V2_XNNPACK_FP32, COREML_FP16: DISTILUSE_BASE_MULTILINGUAL_CASED_V2_COREML_FP16, MLX_INT8: DISTILUSE_BASE_MULTILINGUAL_CASED_V2_MLX_INT8, + VULKAN_FP16: DISTILUSE_BASE_MULTILINGUAL_CASED_V2_VULKAN_FP16, }, // The one Core ML/MLX pair that does not go Core ML's way. On an // iPhone 16, warm, over 150 embeds of five sentences in five languages: From 4629bf4b74d5b10c8c8422f56f9bb0ee69240326 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20S=C5=82uszniak?= Date: Tue, 1 Sep 2026 13:20:02 +0200 Subject: [PATCH 18/24] feat(models): add Vulkan fp16 for Supertonic All four sub-models lower to Vulkan. On a Galaxy S26 Ultra (Adreno 840), medians over interleaved rounds, at 512 text tokens and 1000 latent frames: vulkan xnnpack duration_predictor 18.7 33.2 1.78x text_encoder 72.3 140.9 1.95x vector_estimator 682.5 1253.7 1.84x vocoder 865.8 1284.3 1.48x 2.12x end to end at the default 8 flow-matching steps, where vector_estimator is 84% of the total. Outputs match the fp32 CPU references at cosine 1.000000, 0.999414, 0.999994 and 0.999977. Vulkan leads BACKEND_ORDER.android, so this becomes the Android default. textToSpeech now provisions vulkan in download-libs; without it the backend is never downloaded and the model silently falls back to XNNPACK. Needs the ExecuTorch fixes in pytorch/executorch#22399, #22401, #22402, #22403 and #22406, all cherry-picked into the labs fork and built into the 1.4.1 native libs. --- .../scripts/download-libs.js | 8 ++++++-- packages/react-native-executorch/src/models.ts | 13 +++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/packages/react-native-executorch/scripts/download-libs.js b/packages/react-native-executorch/scripts/download-libs.js index 5c97a39cc1..1ae5d46af4 100644 --- a/packages/react-native-executorch/scripts/download-libs.js +++ b/packages/react-native-executorch/scripts/download-libs.js @@ -131,8 +131,12 @@ const FEATURE_MAP = { privacyFilter: { backends: ['xnnpack', 'mlx'], libs: [] }, // Whisper ships xnnpack, coreml, an MLX iOS export and a Vulkan Android one. speechToText: { backends: ['xnnpack', 'coreml', 'mlx', 'vulkan'], libs: [] }, - // Kokoro ships xnnpack + coreml; Supertonic adds an MLX iOS export. - textToSpeech: { backends: ['xnnpack', 'coreml', 'mlx'], libs: ['phonemis'] }, + // Kokoro ships xnnpack + coreml; Supertonic adds an MLX iOS export and a + // Vulkan Android one. + textToSpeech: { + backends: ['xnnpack', 'coreml', 'mlx', 'vulkan'], + libs: ['phonemis'], + }, // FSMN VAD — xnnpack only. vad: { backends: ['xnnpack'], libs: [] }, // The MiniLM/CLIP-text/distiluse family ships coreml alongside xnnpack, diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts index 3e75228c90..c4b52d532b 100644 --- a/packages/react-native-executorch/src/models.ts +++ b/packages/react-native-executorch/src/models.ts @@ -1116,6 +1116,18 @@ const SUPERTONIC_3_MLX_FP32: SupertonicTtsModel = { voiceStyles: SUPERTONIC_DEFAULT_VOICE_STYLES, }; +const SUPERTONIC_3_VULKAN_FP16: SupertonicTtsModel = { + name: 'supertonic', + modelPaths: { + durationPredictor: `${BASE_URL}-supertonic/${NEXT_VERSION_TAG}/vulkan/duration_predictor_vulkan_fp16.pte`, + vectorEstimator: `${BASE_URL}-supertonic/${NEXT_VERSION_TAG}/vulkan/vector_estimator_vulkan_fp16.pte`, + textEncoder: `${BASE_URL}-supertonic/${NEXT_VERSION_TAG}/vulkan/text_encoder_vulkan_fp16.pte`, + vocoder: `${BASE_URL}-supertonic/${NEXT_VERSION_TAG}/vulkan/vocoder_vulkan_fp16.pte`, + }, + unicodeIndexerPath: `${BASE_URL}-supertonic/${NEXT_VERSION_TAG}/unicode_indexer.json`, + voiceStyles: SUPERTONIC_DEFAULT_VOICE_STYLES, +}; + const KOKORO_ROOT = `${BASE_URL}-kokoro/${NEXT_VERSION_TAG}`; const KOKORO_PHONEMIZER_ROOT = `${KOKORO_ROOT}/phonemizer`; @@ -2624,6 +2636,7 @@ export const models = { SUPERTONIC: variants({ XNNPACK_FP32: SUPERTONIC_3_XNNPACK_FP32, MLX_FP32: SUPERTONIC_3_MLX_FP32, + VULKAN_FP16: SUPERTONIC_3_VULKAN_FP16, }), /** From 5bef3409d3c01e1cde5bdc34cb3edadf41ad24fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20S=C5=82uszniak?= Date: Tue, 1 Sep 2026 13:26:21 +0200 Subject: [PATCH 19/24] feat(speech): offer the Vulkan Supertonic build in the demo Adds the Vulkan entry to the model picker and makes it the initial selection on Android, which is also what SUPERTONIC.DEFAULT resolves to there. --- apps/speech/app/text-to-speech/index.tsx | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/apps/speech/app/text-to-speech/index.tsx b/apps/speech/app/text-to-speech/index.tsx index b6dcb2be1b..1e041a4a24 100644 --- a/apps/speech/app/text-to-speech/index.tsx +++ b/apps/speech/app/text-to-speech/index.tsx @@ -53,11 +53,14 @@ const STEPS_OPTIONS = [ const MODEL_OPTIONS = [ { label: 'XNNPACK (CPU)', value: 'XNNPACK_FP32' as const }, { label: 'MLX (Apple Silicon)', value: 'MLX_FP32' as const, disabled: Platform.OS !== 'ios' }, + { label: 'Vulkan (GPU)', value: 'VULKAN_FP16' as const, disabled: Platform.OS !== 'android' }, ]; function TTSContent() { const [text, setText] = useState(SAMPLE_TEXT); - const [selectedModel, setSelectedModel] = useState<'XNNPACK_FP32' | 'MLX_FP32'>('XNNPACK_FP32'); + const [selectedModel, setSelectedModel] = useState<'XNNPACK_FP32' | 'MLX_FP32' | 'VULKAN_FP16'>( + Platform.OS === 'android' ? 'VULKAN_FP16' : 'XNNPACK_FP32' + ); const [selectedVoice, setSelectedVoice] = useState('F1'); const [selectedLang, setSelectedLang] = useState('en'); const [speed, setSpeed] = useState(1.05); From 1bf7dfff95faca71a2b0f370418c82090aa9ac91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20S=C5=82uszniak?= Date: Wed, 2 Sep 2026 09:27:59 +0200 Subject: [PATCH 20/24] Address review: inline the variant helpers, restore the kokoro paths shape, default whisper base.en/small.en to int8 - fold modelVariants.ts into models.ts as local helpers and reframe its unit tests as registry-level assertions - restore kokoroModelPaths(backend, variant, dir) and the literal Core ML presets from the kokoro review - move the kokoro Core ML rationale out of the public TSDoc block - trim the RF-DETR, whisper and distiluse notes to their gist - put XNNPACK_INT8 ahead of XNNPACK_FP32 for whisper base.en and small.en: over 250 LibriSpeech test-clean clips int8 moves base.en 4.84%% -> 5.20%% WER and small.en 3.42%% -> 3.38%%, too little to outweigh halving the download. tiny.en keeps fp32 first, where int8 costs 6.08%% -> 7.77%% --- .cspell-wordlist.txt | 1 + .../__tests__/api/modelVariants.test.ts | 58 +--- .../src/modelVariants.ts | 173 ---------- .../react-native-executorch/src/models.ts | 315 +++++++++++++----- 4 files changed, 244 insertions(+), 303 deletions(-) delete mode 100644 packages/react-native-executorch/src/modelVariants.ts diff --git a/.cspell-wordlist.txt b/.cspell-wordlist.txt index 58d46387fd..051b033392 100644 --- a/.cspell-wordlist.txt +++ b/.cspell-wordlist.txt @@ -350,3 +350,4 @@ phonemizes həlˈoʊ NSURL backgrounding +LibriSpeech diff --git a/packages/react-native-executorch/__tests__/api/modelVariants.test.ts b/packages/react-native-executorch/__tests__/api/modelVariants.test.ts index 8a93f6fb4a..93b1e229a0 100644 --- a/packages/react-native-executorch/__tests__/api/modelVariants.test.ts +++ b/packages/react-native-executorch/__tests__/api/modelVariants.test.ts @@ -294,57 +294,35 @@ describe('DEFAULT variant resolution', () => { }); }); -describe('variants()', () => { - const A = { modelPath: 'a.pte' }; - const B = { modelPath: 'b.pte' }; - const C = { modelPath: 'c.pte' }; - - /** - * Reloads the helper module for a given device. - * @param os The platform to resolve for. - * @param backends The backends the binary reports as linked. - * @returns The freshly loaded `modelVariants` module. - */ - function load(os: 'ios' | 'android', backends?: string[]) { - fakeJsi.setRegisteredBackends(backends ?? ['XnnpackBackend', 'CoreMLBackend']); - jest.resetModules(); - setPlatform(os); - return require('../../src/modelVariants') as typeof import('../../src/modelVariants'); - } +describe('variant selection rules', () => { + /** The distiluse group, the one place a pin overrides the backend order. */ + const distiluse = (registry: Node): Node => + (registry.textEmbeddings as Node).DISTILUSE_BASE_MULTILINGUAL_CASED_V2 as Node; it('breaks a tie within one backend by declaration order', () => { - const { variants } = load('android'); - expect(variants({ XNNPACK_INT8: A, XNNPACK_FP32: B }).DEFAULT).toBe(A); - expect(variants({ XNNPACK_FP32: B, XNNPACK_INT8: A }).DEFAULT).toBe(B); + // With XNNPACK the only linked backend, every group has to land on the + // first XNNPACK variant it declares, whatever else it publishes. + const offenders = defaultsOf(registryFor({ os: 'ios', backends: ['XnnpackBackend'] })) + .filter(({ offers }) => offers.some((key) => key.startsWith('XNNPACK'))) + .filter(({ key, offers }) => key !== offers.find((entry) => entry.startsWith('XNNPACK'))) + .map(({ label, key, offers }) => `${label}: ${key} of ${offers.join(', ')}`); + + expect(offenders).toEqual([]); }); it('honours a pinned variant over the backend order', () => { - const { variants } = load('ios'); - const group = variants({ XNNPACK_FP32: A, COREML_FP16: B }, { ios: 'XNNPACK_FP32' }); - expect(group.DEFAULT).toBe(A); + // Core ML leads the iOS order, so MLX here is the pin and nothing else. + expect(defaultKeyOf(distiluse(registryFor({ os: 'ios' })))).toBe('MLX_INT8'); }); it('ignores a pin whose backend the app did not link in', () => { - const { variants } = load('ios', ['XnnpackBackend']); - const group = variants({ XNNPACK_FP32: A, COREML_FP16: B }, { ios: 'COREML_FP16' }); - expect(group.DEFAULT).toBe(A); + const registry = registryFor({ os: 'ios', backends: ['XnnpackBackend'] }); + expect(defaultKeyOf(distiluse(registry))).toBe('XNNPACK_8DA4W'); }); it('applies a pin only on the platform it names', () => { - const pinned = { android: 'XNNPACK_INT8' } as const; - expect(load('android').variants({ XNNPACK_FP32: A, XNNPACK_INT8: B }, pinned).DEFAULT).toBe(B); - expect(load('ios').variants({ XNNPACK_FP32: A, XNNPACK_INT8: B }, pinned).DEFAULT).toBe(A); - }); - - it('falls back to the first variant when no preferred backend is linked in', () => { - const { variants } = load('android', ['CoreMLBackend']); - expect(variants({ COREML_FP16: C, XNNPACK_FP32: A }).DEFAULT).toBe(C); - }); - - it('keeps every named variant alongside the default', () => { - const { variants } = load('ios'); - const group = variants({ XNNPACK_FP32: A, COREML_FP16: B }); - expect(group).toEqual({ XNNPACK_FP32: A, COREML_FP16: B, DEFAULT: B }); + // The same group pins MLX for iOS only; Android resolves by its own order. + expect(defaultKeyOf(distiluse(registryFor({ os: 'android' })))).toBe('VULKAN_FP16'); }); }); diff --git a/packages/react-native-executorch/src/modelVariants.ts b/packages/react-native-executorch/src/modelVariants.ts deleted file mode 100644 index 5775f1f729..0000000000 --- a/packages/react-native-executorch/src/modelVariants.ts +++ /dev/null @@ -1,173 +0,0 @@ -/** - * Platform-aware resolution of the `DEFAULT` alias in the {@link models} - * registry. - * - * Every model in the registry that ships more than one export lists its - * variants under backend-tagged keys (`XNNPACK_INT8`, `COREML_FP16`, ...) and - * exposes a `DEFAULT` alias next to them. Pinning that alias to one fixed - * variant means most users silently run the universal XNNPACK build on - * hardware that has a much faster accelerator sitting idle — Core ML on iOS in - * particular. `DEFAULT` is therefore resolved here, once at import time, from: - * - * 1. the platform the app is running on, - * 2. the backends actually linked into the binary — the install-time - * `react-native-executorch` block in the app's `package.json` decides these, - * and every backend is on unless the app opts out, - * 3. the order the variants are declared in, which breaks ties within a - * backend (`XNNPACK_INT8` before `XNNPACK_FP32` means "int8 unless told - * otherwise"). - * - * A model whose best variant does not follow from that ordering can pin one - * per platform — see the second argument of {@link variants}. A model that - * publishes both a Core ML and an MLX export has to: the two are close enough - * that the winner is a per-model benchmark result, not something an ordering - * can state, so `every group offering both Core ML and MLX pins one` in - * `__tests__/api/modelVariants.test.ts` fails until the choice is written down. - * @module ModelVariants - * @internal - */ - -import { Platform } from 'react-native'; - -import { rnexecutorchJsi } from './native/bridge'; - -/** The backend prefix a variant key starts with. */ -type BackendTag = 'XNNPACK' | 'COREML' | 'MLX' | 'VULKAN'; - -/** The platforms the registry resolves defaults for. */ -type TargetPlatform = 'ios' | 'android'; - -/** - * Backends to try, best first, per platform. - * - * The accelerated backends lead and XNNPACK trails on both platforms: a model - * is only exported to Core ML, MLX or Vulkan once it has been shown to run - * better there, so a published accelerated variant is itself the signal that it - * should be preferred. XNNPACK is the fallback because it is the one backend - * every model exports to. - * - * Core ML sits above MLX on iOS only to give the pair a deterministic order; - * every model that publishes both has to pin its winner explicitly. - */ -const BACKEND_ORDER: Record = { - ios: ['COREML', 'MLX', 'XNNPACK'], - android: ['VULKAN', 'XNNPACK'], -}; - -/** Variant keys pinned per platform, overriding {@link BACKEND_ORDER}. */ -type PinnedVariants = Partial>>; - -const ALL_TAGS: readonly BackendTag[] = ['XNNPACK', 'COREML', 'MLX', 'VULKAN']; - -/** - * Narrows `Platform.OS` to the platforms the registry distinguishes. - * @returns The platform to resolve defaults for. - */ -function currentPlatform(): TargetPlatform { - return Platform.OS === 'ios' ? 'ios' : 'android'; -} - -/** - * Backends that are linked into this binary and usable on this device. - * @returns The usable backend tags — every one of them when the native runtime - * cannot be asked, so that a missing answer widens the choice rather than - * narrowing it to nothing. - */ -function usableBackends(): ReadonlySet { - let registered: readonly string[] = []; - try { - registered = rnexecutorchJsi.getExecuTorchRegisteredBackends(); - } catch { - registered = []; - } - if (registered.length === 0) return new Set(ALL_TAGS); - - const names = registered.map((name) => name.toLowerCase()); - const usable = ALL_TAGS.filter((tag) => names.some((name) => name.startsWith(tag.toLowerCase()))); - - // The simulator links the Core ML backend but cannot run it: it has no - // Neural Engine, and MPSGraph refuses the compiled models outright. MLX only - // ever ships a device slice, so it drops out of `registered` on its own. - if (currentPlatform() === 'ios' && rnexecutorchJsi.isEmulator === true) { - return new Set(usable.filter((tag) => tag !== 'COREML' && tag !== 'MLX')); - } - return new Set(usable); -} - -const PLATFORM = currentPlatform(); -const USABLE = usableBackends(); - -/** - * Reads the backend out of a variant key. - * @param key The variant key, e.g. `COREML_FP16`. - * @returns The backend the key names, or `undefined` when it names none. - */ -function backendOf(key: string): BackendTag | undefined { - return ALL_TAGS.find((tag) => key === tag || key.startsWith(`${tag}_`)); -} - -/** - * Picks the variant key this platform should default to. - * @param keys The group's variant keys, in declaration order. - * @param pinned Per-platform overrides. - * @returns The chosen key. - */ -function pickVariant( - keys: readonly string[], - pinned?: PinnedVariants> -): string { - const pin = pinned?.[PLATFORM]; - const pinnedBackend = pin === undefined ? undefined : backendOf(pin); - if (pin !== undefined && keys.includes(pin) && pinnedBackend && USABLE.has(pinnedBackend)) { - return pin; - } - - for (const tag of BACKEND_ORDER[PLATFORM]) { - if (!USABLE.has(tag)) continue; - const match = keys.find((key) => backendOf(key) === tag); - if (match !== undefined) return match; - } - - // No preferred backend is both published for this model and linked into the - // build — an app that opted out of the backends its models need. Hand back - // the first variant so the registry still names a model and the failure - // surfaces at load, where the error says which backend is missing. - return keys[0]!; -} - -/** - * Adds a platform-resolved `DEFAULT` to a group of backend variants. - * - * Declare the variants best-first within each backend: with several exports - * from the same backend, the earliest one wins. - * @typeParam V The variant map. - * @param map The group's variants, keyed by backend and precision. - * @param pinned Variant keys to prefer on a given platform, for models whose - * best export does not follow from the declaration order. Ignored when the - * pinned variant's backend is not linked into the build. - * @returns The variants, plus the `DEFAULT` alias for this platform. - */ -export function variants>( - map: V, - pinned?: PinnedVariants -): V & { readonly DEFAULT: V[keyof V] } { - const key = pickVariant(Object.keys(map), pinned); - return { ...map, DEFAULT: map[key] as V[keyof V] }; -} - -/** - * Adds a `DEFAULT` to a group of sub-groups — a model family split by scale or - * input size — mirroring the `DEFAULT` of the first sub-group declared. - * - * The sub-group resolved its own default per platform, so the family inherits - * that without repeating the rules. - * @typeParam V The sub-group map. - * @param map The family's sub-groups, most representative first. - * @returns The sub-groups, plus the inherited `DEFAULT`. - */ -export function family>( - map: V -): V & { readonly DEFAULT: V[keyof V]['DEFAULT'] } { - const first = Object.keys(map)[0]!; - return { ...map, DEFAULT: map[first]!.DEFAULT as V[keyof V]['DEFAULT'] }; -} diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts index 4f0b0e2fde..6d0e66eb91 100644 --- a/packages/react-native-executorch/src/models.ts +++ b/packages/react-native-executorch/src/models.ts @@ -7,13 +7,15 @@ * URLs, tokenizer/phonemizer files, preprocessing parameters, and label maps. * * A model that ships several exports lists them under backend-tagged keys and - * wraps the group in `variants`, which adds the `DEFAULT` alias that resolves - * to the fastest export the current platform can run — see `modelVariants.ts`. - * Within one backend the first variant declared wins, so keep the group - * ordered best-first. + * wraps the group in `variants`, which adds the `DEFAULT` alias resolving to + * the fastest export the current platform can run. Within one backend the + * first variant declared wins, so keep the group ordered best-first. * @module Models */ +import { Platform } from 'react-native'; + +import { rnexecutorchJsi } from './native/bridge'; import type { ClassifierModel } from './extensions/cv/tasks/classification'; import type { ObjectDetectorModel } from './extensions/cv/tasks/objectDetection'; import type { StyleTransferModel } from './extensions/cv/tasks/styleTransfer'; @@ -34,7 +36,6 @@ import { } from './extensions/speech/tasks/whisperSpeechToText'; import type { PaddleOcrModel } from './extensions/cv/tasks/paddleOcr'; import type { LLMModel } from './extensions/llm/tasks/llmChatSession'; -import { variants, family } from './modelVariants'; import { IMAGENET_NORM, IMAGENET1K_LABELS, @@ -57,6 +58,127 @@ import { type SupertonicDefaultVoiceName, } from './constants'; +// ============================================================================= +// DEFAULT variant resolution +// ============================================================================= +// `DEFAULT` is resolved once, when this module is first imported, from the +// platform, the backends the binary was linked with, and the order the +// variants are declared in. A group whose best export does not follow from +// that order pins one per platform — see the second argument of `variants`. + +/** The backend prefix a variant key starts with. */ +type BackendTag = 'XNNPACK' | 'COREML' | 'MLX' | 'VULKAN'; + +/** The platforms the registry resolves defaults for. */ +type TargetPlatform = 'ios' | 'android'; + +const ALL_BACKENDS: readonly BackendTag[] = ['XNNPACK', 'COREML', 'MLX', 'VULKAN']; + +// Accelerators lead and XNNPACK trails: a model is only exported to Core ML, +// MLX or Vulkan once it has been shown to run better there, and XNNPACK is the +// one backend every model exports to. Core ML sits above MLX only to make the +// order deterministic; every group publishing both pins its winner explicitly. +const BACKEND_ORDER: Record = { + ios: ['COREML', 'MLX', 'XNNPACK'], + android: ['VULKAN', 'XNNPACK'], +}; + +const PLATFORM: TargetPlatform = Platform.OS === 'ios' ? 'ios' : 'android'; + +/** + * Backends linked into this binary and usable on this device. + * @returns The usable tags — every one of them when the native runtime cannot + * be asked, so that a missing answer widens the choice rather than emptying it. + */ +function usableBackends(): ReadonlySet { + let registered: readonly string[] = []; + try { + registered = rnexecutorchJsi.getExecuTorchRegisteredBackends(); + } catch { + registered = []; + } + if (registered.length === 0) return new Set(ALL_BACKENDS); + + const names = registered.map((name) => name.toLowerCase()); + const usable = ALL_BACKENDS.filter((tag) => + names.some((name) => name.startsWith(tag.toLowerCase())) + ); + + // The simulator links the Core ML backend but cannot run it: no Neural + // Engine, and MPSGraph refuses the compiled models. MLX only ever ships a + // device slice, so it drops out of `registered` on its own. + const onSimulator = PLATFORM === 'ios' && rnexecutorchJsi.isEmulator === true; + return new Set(onSimulator ? usable.filter((tag) => tag !== 'COREML' && tag !== 'MLX') : usable); +} + +const USABLE_BACKENDS = usableBackends(); + +/** + * Reads the backend out of a variant key. + * @param key The variant key, e.g. `COREML_FP16`. + * @returns The backend the key names, or `undefined` when it names none. + */ +const backendOf = (key: string): BackendTag | undefined => + ALL_BACKENDS.find((tag) => key === tag || key.startsWith(`${tag}_`)); + +/** + * Picks the variant key this platform should default to. + * @param keys The group's variant keys, in declaration order. + * @param pin The key pinned for this platform, if any. + * @returns The chosen key. + */ +function pickVariant(keys: readonly string[], pin?: string): string { + const pinned = pin === undefined ? undefined : backendOf(pin); + if (pin !== undefined && keys.includes(pin) && pinned && USABLE_BACKENDS.has(pinned)) return pin; + + for (const tag of BACKEND_ORDER[PLATFORM]) { + if (!USABLE_BACKENDS.has(tag)) continue; + const match = keys.find((key) => backendOf(key) === tag); + if (match !== undefined) return match; + } + + // No preferred backend is both published for this model and linked into the + // build — an app that opted out of the backends its models need. Hand back + // the first variant so the registry still names a model and the failure + // surfaces at load, where the error says which backend is missing. + return keys[0]!; +} + +/** + * Adds a platform-resolved `DEFAULT` to a group of backend variants. + * + * Declare the variants best-first within each backend: with several exports + * from the same backend, the earliest one wins. + * @typeParam V The variant map. + * @param map The group's variants, keyed by backend and precision. + * @param pinned Variant keys to prefer on a given platform, for groups whose + * best export does not follow from the declaration order. Ignored when the + * pinned variant's backend is not linked into the build. + * @returns The variants, plus the `DEFAULT` alias for this platform. + */ +function variants>( + map: V, + pinned?: Partial>> +): V & { readonly DEFAULT: V[keyof V] } { + const key = pickVariant(Object.keys(map), pinned?.[PLATFORM]); + return { ...map, DEFAULT: map[key] as V[keyof V] }; +} + +/** + * Adds a `DEFAULT` to a group of sub-groups — a model family split by scale or + * input size — mirroring the `DEFAULT` of the first sub-group declared, which + * resolved itself per platform. + * @typeParam V The sub-group map. + * @param map The family's sub-groups, most representative first. + * @returns The sub-groups, plus the inherited `DEFAULT`. + */ +function family>( + map: V +): V & { readonly DEFAULT: V[keyof V]['DEFAULT'] } { + const first = Object.keys(map)[0]!; + return { ...map, DEFAULT: map[first]!.DEFAULT as V[keyof V]['DEFAULT'] }; +} + const BASE_URL = 'https://huggingface.co/software-mansion/react-native-executorch'; const VERSION_TAG = 'resolve/v0.9.0'; const NEXT_VERSION_TAG = 'resolve/v0.10.0'; @@ -1131,30 +1253,15 @@ const SUPERTONIC_3_VULKAN_FP16: SupertonicTtsModel = const KOKORO_ROOT = `${BASE_URL}-kokoro/${NEXT_VERSION_TAG}`; const KOKORO_PHONEMIZER_ROOT = `${KOKORO_ROOT}/phonemizer`; -// Both backends export at fp32, so the precision is not a parameter here. const kokoroModelPaths = ( + backend: 'xnnpack' | 'coreml', variant: 'std' | 'pl' | 'de', - dir: string, - backend: 'xnnpack' | 'coreml' = 'xnnpack' + dir: string ) => ({ durationPredictor: `${KOKORO_ROOT}/${backend}/${dir}/duration_predictor_${variant}_${backend}_fp32.pte`, synthesizer: `${KOKORO_ROOT}/${backend}/${dir}/synthesizer_${variant}_${backend}_fp32.pte`, }); -const KOKORO_STANDARD_PATHS = kokoroModelPaths('std', 'standard'); -const KOKORO_POLISH_PATHS = kokoroModelPaths('pl', 'polish'); -const KOKORO_GERMAN_PATHS = kokoroModelPaths('de', 'german'); -const KOKORO_STANDARD_COREML_PATHS = kokoroModelPaths('std', 'standard', 'coreml'); -const KOKORO_POLISH_COREML_PATHS = kokoroModelPaths('pl', 'polish', 'coreml'); -const KOKORO_GERMAN_COREML_PATHS = kokoroModelPaths('de', 'german', 'coreml'); - -// A language differs from its Core ML twin only in which pair of .pte files it -// loads: the voices, the phonemizer and the voice union are all backend-free. -const kokoroCoreMl = ( - model: KokoroTtsModel, - modelPaths: KokoroTtsModel['modelPaths'] -): KokoroTtsModel => ({ ...model, modelPaths }); - const kokoroVoices = (names: readonly N[]) => names.reduce( (acc, name) => ({ ...acc, [name]: `${KOKORO_ROOT}/voices/${name}.bin` }), @@ -1181,74 +1288,119 @@ const KOKORO_EN_US_XNNPACK_FP32: KokoroTtsModel< 'af_heart' | 'af_river' | 'af_sarah' | 'am_adam' | 'am_michael' | 'am_santa' > = { name: 'kokoro', - modelPaths: KOKORO_STANDARD_PATHS, + modelPaths: kokoroModelPaths('xnnpack', 'std', 'standard'), phonemizer: kokoroEnglishPhonemizer('en-us'), voices: kokoroVoices(['af_heart', 'af_river', 'af_sarah', 'am_adam', 'am_michael', 'am_santa']), }; const KOKORO_EN_GB_XNNPACK_FP32: KokoroTtsModel<'bf_emma' | 'bm_daniel'> = { name: 'kokoro', - modelPaths: KOKORO_STANDARD_PATHS, + modelPaths: kokoroModelPaths('xnnpack', 'std', 'standard'), phonemizer: kokoroEnglishPhonemizer('en-gb'), voices: kokoroVoices(['bf_emma', 'bm_daniel']), }; const KOKORO_ES_XNNPACK_FP32: KokoroTtsModel<'ef_dora' | 'em_alex'> = { name: 'kokoro', - modelPaths: KOKORO_STANDARD_PATHS, + modelPaths: kokoroModelPaths('xnnpack', 'std', 'standard'), phonemizer: kokoroNeuralPhonemizer('es'), voices: kokoroVoices(['ef_dora', 'em_alex']), }; const KOKORO_FR_XNNPACK_FP32: KokoroTtsModel<'ff_siwis'> = { name: 'kokoro', - modelPaths: KOKORO_STANDARD_PATHS, + modelPaths: kokoroModelPaths('xnnpack', 'std', 'standard'), phonemizer: kokoroNeuralPhonemizer('fr'), voices: kokoroVoices(['ff_siwis']), }; const KOKORO_IT_XNNPACK_FP32: KokoroTtsModel<'if_sara' | 'im_nicola'> = { name: 'kokoro', - modelPaths: KOKORO_STANDARD_PATHS, + modelPaths: kokoroModelPaths('xnnpack', 'std', 'standard'), phonemizer: kokoroNeuralPhonemizer('it'), voices: kokoroVoices(['if_sara', 'im_nicola']), }; const KOKORO_PT_XNNPACK_FP32: KokoroTtsModel<'pf_dora' | 'pm_santa'> = { name: 'kokoro', - modelPaths: KOKORO_STANDARD_PATHS, + modelPaths: kokoroModelPaths('xnnpack', 'std', 'standard'), phonemizer: kokoroNeuralPhonemizer('pt'), voices: kokoroVoices(['pf_dora', 'pm_santa']), }; const KOKORO_HI_XNNPACK_FP32: KokoroTtsModel<'hf_alpha' | 'hm_omega' | 'hm_psi'> = { name: 'kokoro', - modelPaths: KOKORO_STANDARD_PATHS, + modelPaths: kokoroModelPaths('xnnpack', 'std', 'standard'), phonemizer: kokoroNeuralPhonemizer('hi'), voices: kokoroVoices(['hf_alpha', 'hm_omega', 'hm_psi']), }; const KOKORO_PL_XNNPACK_FP32: KokoroTtsModel<'pm_mateusz'> = { name: 'kokoro', - modelPaths: KOKORO_POLISH_PATHS, + modelPaths: kokoroModelPaths('xnnpack', 'pl', 'polish'), phonemizer: kokoroNeuralPhonemizer('pl'), voices: kokoroVoices(['pm_mateusz']), }; const KOKORO_DE_XNNPACK_FP32: KokoroTtsModel<'df_anna'> = { name: 'kokoro', - modelPaths: KOKORO_GERMAN_PATHS, + modelPaths: kokoroModelPaths('xnnpack', 'de', 'german'), phonemizer: kokoroNeuralPhonemizer('de'), voices: kokoroVoices(['df_anna']), }; -const KOKORO_EN_US_COREML_FP32 = kokoroCoreMl( - KOKORO_EN_US_XNNPACK_FP32, - KOKORO_STANDARD_COREML_PATHS -); -const KOKORO_EN_GB_COREML_FP32 = kokoroCoreMl( - KOKORO_EN_GB_XNNPACK_FP32, - KOKORO_STANDARD_COREML_PATHS -); -const KOKORO_ES_COREML_FP32 = kokoroCoreMl(KOKORO_ES_XNNPACK_FP32, KOKORO_STANDARD_COREML_PATHS); -const KOKORO_FR_COREML_FP32 = kokoroCoreMl(KOKORO_FR_XNNPACK_FP32, KOKORO_STANDARD_COREML_PATHS); -const KOKORO_IT_COREML_FP32 = kokoroCoreMl(KOKORO_IT_XNNPACK_FP32, KOKORO_STANDARD_COREML_PATHS); -const KOKORO_PT_COREML_FP32 = kokoroCoreMl(KOKORO_PT_XNNPACK_FP32, KOKORO_STANDARD_COREML_PATHS); -const KOKORO_HI_COREML_FP32 = kokoroCoreMl(KOKORO_HI_XNNPACK_FP32, KOKORO_STANDARD_COREML_PATHS); -const KOKORO_PL_COREML_FP32 = kokoroCoreMl(KOKORO_PL_XNNPACK_FP32, KOKORO_POLISH_COREML_PATHS); -const KOKORO_DE_COREML_FP32 = kokoroCoreMl(KOKORO_DE_XNNPACK_FP32, KOKORO_GERMAN_COREML_PATHS); +// Core ML counterparts: the same weights, phonemizer and voices as the XNNPACK +// presets, only the `.pte` files differ. Their token axis is fixed at 128, so +// the pipeline pads every chunk up to it. iOS only. +const KOKORO_EN_US_COREML_FP32: KokoroTtsModel< + 'af_heart' | 'af_river' | 'af_sarah' | 'am_adam' | 'am_michael' | 'am_santa' +> = { + name: 'kokoro', + modelPaths: kokoroModelPaths('coreml', 'std', 'standard'), + phonemizer: kokoroEnglishPhonemizer('en-us'), + voices: kokoroVoices(['af_heart', 'af_river', 'af_sarah', 'am_adam', 'am_michael', 'am_santa']), +}; +const KOKORO_EN_GB_COREML_FP32: KokoroTtsModel<'bf_emma' | 'bm_daniel'> = { + name: 'kokoro', + modelPaths: kokoroModelPaths('coreml', 'std', 'standard'), + phonemizer: kokoroEnglishPhonemizer('en-gb'), + voices: kokoroVoices(['bf_emma', 'bm_daniel']), +}; +const KOKORO_ES_COREML_FP32: KokoroTtsModel<'ef_dora' | 'em_alex'> = { + name: 'kokoro', + modelPaths: kokoroModelPaths('coreml', 'std', 'standard'), + phonemizer: kokoroNeuralPhonemizer('es'), + voices: kokoroVoices(['ef_dora', 'em_alex']), +}; +const KOKORO_FR_COREML_FP32: KokoroTtsModel<'ff_siwis'> = { + name: 'kokoro', + modelPaths: kokoroModelPaths('coreml', 'std', 'standard'), + phonemizer: kokoroNeuralPhonemizer('fr'), + voices: kokoroVoices(['ff_siwis']), +}; +const KOKORO_IT_COREML_FP32: KokoroTtsModel<'if_sara' | 'im_nicola'> = { + name: 'kokoro', + modelPaths: kokoroModelPaths('coreml', 'std', 'standard'), + phonemizer: kokoroNeuralPhonemizer('it'), + voices: kokoroVoices(['if_sara', 'im_nicola']), +}; +const KOKORO_PT_COREML_FP32: KokoroTtsModel<'pf_dora' | 'pm_santa'> = { + name: 'kokoro', + modelPaths: kokoroModelPaths('coreml', 'std', 'standard'), + phonemizer: kokoroNeuralPhonemizer('pt'), + voices: kokoroVoices(['pf_dora', 'pm_santa']), +}; +const KOKORO_HI_COREML_FP32: KokoroTtsModel<'hf_alpha' | 'hm_omega' | 'hm_psi'> = { + name: 'kokoro', + modelPaths: kokoroModelPaths('coreml', 'std', 'standard'), + phonemizer: kokoroNeuralPhonemizer('hi'), + voices: kokoroVoices(['hf_alpha', 'hm_omega', 'hm_psi']), +}; + +const KOKORO_PL_COREML_FP32: KokoroTtsModel<'pm_mateusz'> = { + name: 'kokoro', + modelPaths: kokoroModelPaths('coreml', 'pl', 'polish'), + phonemizer: kokoroNeuralPhonemizer('pl'), + voices: kokoroVoices(['pm_mateusz']), +}; +const KOKORO_DE_COREML_FP32: KokoroTtsModel<'df_anna'> = { + name: 'kokoro', + modelPaths: kokoroModelPaths('coreml', 'de', 'german'), + phonemizer: kokoroNeuralPhonemizer('de'), + voices: kokoroVoices(['df_anna']), +}; // ============================================================================= // Privacy Filter @@ -1931,24 +2083,11 @@ export const models = { MLX_FP32: RFDETR_KEYPOINT_MLX_FP32, }, // Core ML over MLX: 144.0 ms against 272.3 on an iPhone 16, at 263 MB - // against 1304 MB. - // - // fp16 is the only Core ML build published for this model. It replaced an - // fp32 one that was 165.1 ms, 389 MB peak and twice the download, after - // fp16 was checked against it across 13 real photos and 40 detections: - // visible landmarks agreed to 1.04 px, boxes to 0.653 px and scores to - // 0.072, with no detection crossing the threshold. (Landmarks the model - // marks invisible drift further, up to 15.7 px, but their coordinates are - // undefined when `vis` is 0.) - // - // That fp16 build took a while to exist and is fragile in a specific way, - // so do not "simplify" it. It is correct on device only with both the CPU - // and the Neural Engine excluded — measured against fp32's 0.863, `all` - // gives 0.421, `cpu_only` 0.411, `cpu_and_ne` 0.422 and `cpu_and_gpu` - // 0.862 — and on macOS only the CPU path is wrong, so a host check passes - // a build the device gets wrong. It also needs an iOS17 deployment target; - // iOS18 degrades every build of this model at either precision. Both - // constraints live in export-scripts (MR !18). + // against 1304 MB. fp16 matches the fp32 build it replaced (landmarks to + // 1.04 px over 13 photos) but only under the GPU-only compute unit and an + // iOS17 deployment target — every other combination degrades it, and a + // macOS check passes builds the device gets wrong. See export-scripts + // MR !18 before re-exporting. { ios: 'COREML_FP16' } ), }, @@ -2112,16 +2251,10 @@ export const models = { * Voice Activity Detection. Includes multilingual and English-only (`EN`) * variants across model sizes (`TINY`, `BASE`, `SMALL`). */ - // Every size defaults to Core ML over MLX on iOS, which wins on all three - // axes. Speed: 2.5-3.1x faster end to end on 14.5s of real speech, once the - // Core ML builds stopped dispatching their single-token `decode` to the - // Neural Engine (export-scripts MR !18 — decode is dispatch-bound, so - // CPU_ONLY runs it at 6.7 ms/step against 20.0). Memory: base peaks at - // 532 MB against 1834 MB for MLX bf16, which is jetsam territory on an - // iPhone 16 — MLX bf16 at small does not load at all. Accuracy: on the same - // clip MLX bf16 misheard "brown fox" as "round box" and "compute unit" as - // "computer unit", where Core ML matched the reference exactly. - // Reach for MLX_INT8 explicitly if you want the GPU path. + // Every size defaults to Core ML over MLX on iOS: 2.5-3.1x faster end to + // end on an iPhone 16, a third of the peak memory (MLX bf16 at `SMALL` does + // not load at all), and more accurate on the same clip. Reach for MLX_INT8 + // explicitly if you want the GPU path. WHISPER: { /** * Multilingual Whisper Tiny model. Supporting 99+ languages. High speed @@ -2171,6 +2304,13 @@ export const models = { { ios: 'COREML_FP16' } ), /** English-only optimized Whisper models (`TINY`, `BASE`, `SMALL`). */ + // The only sizes with an XNNPACK int8 export, and int8 leads fp32 for all + // but `TINY`. Greedy-decoding 250 LibriSpeech test-clean clips (31 min, + // ~4600 words) through this pipeline, int8 moves base.en 4.84% -> 5.20% + // WER and small.en 3.42% -> 3.38%: too little to outweigh halving the + // download (247 MB against 399, 448 against 1129). `TINY` keeps fp32 + // first because there int8 costs 6.08% -> 7.77%, a quarter of the + // accuracy the smallest model has left. EN: { /** * English-only Whisper Tiny model. Fast and compact for English STT. @@ -2194,8 +2334,8 @@ export const models = { */ BASE: variants( { - XNNPACK_FP32: WHISPER_BASE_EN_XNNPACK_FP32, XNNPACK_INT8: WHISPER_BASE_EN_XNNPACK_INT8, + XNNPACK_FP32: WHISPER_BASE_EN_XNNPACK_FP32, COREML_FP16: WHISPER_BASE_EN_COREML_FP16, MLX_BF16: WHISPER_BASE_EN_MLX_BF16, MLX_INT8: WHISPER_BASE_EN_MLX_INT8, @@ -2211,8 +2351,8 @@ export const models = { */ SMALL: variants( { - XNNPACK_FP32: WHISPER_SMALL_EN_XNNPACK_FP32, XNNPACK_INT8: WHISPER_SMALL_EN_XNNPACK_INT8, + XNNPACK_FP32: WHISPER_SMALL_EN_XNNPACK_FP32, COREML_FP16: WHISPER_SMALL_EN_COREML_FP16, MLX_INT8: WHISPER_SMALL_EN_MLX_INT8, VULKAN_FP16: WHISPER_SMALL_EN_VULKAN_FP16, @@ -2513,13 +2653,10 @@ export const models = { MLX_INT8: DISTILUSE_BASE_MULTILINGUAL_CASED_V2_MLX_INT8, VULKAN_FP16: DISTILUSE_BASE_MULTILINGUAL_CASED_V2_VULKAN_FP16, }, - // The one Core ML/MLX pair that does not go Core ML's way. On an - // iPhone 16, warm, over 150 embeds of five sentences in five languages: - // MLX int8 3.46 ms, Core ML fp16 3.96 ms, XNNPACK 8da4w 5.86 ms, stable - // when the arms are run in reverse. MLX is also half the download - // (140 MB against 271 MB) and skips Core ML's 786 ms first-use compile. - // Fidelity does not break the tie: against the XNNPACK reference both - // sit at 0.973 worst-case cosine. + // The one Core ML/MLX pair that does not go Core ML's way: on an + // iPhone 16, warm, MLX int8 embeds in 3.46 ms against Core ML fp16's + // 3.96, at half the download and no first-use compile, and the two match + // the XNNPACK reference equally well. { ios: 'MLX_INT8' } ), /** @@ -2625,14 +2762,12 @@ export const models = { * Kokoro — a lightweight phoneme-driven Text-to-Speech model. Each language * entry bundles the matching model weights, grapheme-to-phoneme assets and * the voices available for that language, nested per backend. - * - * The Core ML builds are the registry's only fp32 Core ML exports — that is - * what is published, and fp32 keeps them off the Neural Engine, which is - * fp16-only. They still default on iOS because the duration predictor runs - * 14-21x faster warm there than the XNNPACK one on an iPhone 16. The cost - * is a one-time per-method Core ML compile on first use (~13s), cached - * across launches; reach for `XNNPACK_FP32` explicitly to avoid it. */ + // The Core ML builds are the registry's only fp32 Core ML exports, which + // keeps them off the fp16-only Neural Engine, and they still default on + // iOS: the duration predictor runs 14-21x faster warm than the XNNPACK one + // on an iPhone 16. The cost is a one-time ~13s compile on first use, cached + // across launches; reach for `XNNPACK_FP32` explicitly to avoid it. KOKORO: { EN_US: variants({ XNNPACK_FP32: KOKORO_EN_US_XNNPACK_FP32, From 4490098bb8818a86cb8ea0ec022c77c475f0cc95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20S=C5=82uszniak?= Date: Wed, 2 Sep 2026 09:44:43 +0200 Subject: [PATCH 21/24] feat(models): add Vulkan int8 for the mpnet embedders Both mpnet encoders now ship an int8 weight-only Vulkan export alongside the fp16 one: two thirds the download (218 -> 133 MB) and faster on the Adreno 840, at cosine 0.998 against the fp32 reference instead of 0.99997. all-mpnet-base-v2 defaults to it on Android, where the accuracy is flat across the length range (0.9989 at 1 token, 0.9983 at the 382-token bound) and the download saving is worth 0.002 cosine. multi-qa-mpnet-base-dot-v1 keeps fp16 as its default: its int8 error grows with length, 0.9985 down to 0.9952 at the 510-token bound, and it is a dot-product retrieval model, so a systematic similarity shift matters more there. Both variants remain reachable by name; the declaration order within a backend picks the default. The int8 files need the runtime fix in v0.10.0-libs-1.4.1 (pytorch/executorch#22430); an older Vulkan lib aborts on a missing linear_qcs8w_* shader. --- packages/react-native-executorch/src/models.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts index 428128bd17..8aa102dbf7 100644 --- a/packages/react-native-executorch/src/models.ts +++ b/packages/react-native-executorch/src/models.ts @@ -836,6 +836,10 @@ const ALL_MPNET_BASE_V2_EMBEDDINGS: TextEmbedderModel = { modelPath: `${BASE_URL}-all-mpnet-base-v2/${NEXT_VERSION_TAG}/xnnpack/all_mpnet_base_v2_xnnpack_fp32.pte`, tokenizerPath: `${BASE_URL}-all-mpnet-base-v2/${NEXT_VERSION_TAG}/tokenizer.json`, }; +const ALL_MPNET_BASE_V2_VULKAN_INT8: TextEmbedderModel = { + modelPath: `${BASE_URL}-all-mpnet-base-v2/${NEXT_VERSION_TAG}/vulkan/all_mpnet_base_v2_vulkan_int8.pte`, + tokenizerPath: `${BASE_URL}-all-mpnet-base-v2/${NEXT_VERSION_TAG}/tokenizer.json`, +}; const ALL_MPNET_BASE_V2_VULKAN_FP16: TextEmbedderModel = { modelPath: `${BASE_URL}-all-mpnet-base-v2/${NEXT_VERSION_TAG}/vulkan/all_mpnet_base_v2_vulkan_fp16.pte`, tokenizerPath: `${BASE_URL}-all-mpnet-base-v2/${NEXT_VERSION_TAG}/tokenizer.json`, @@ -856,6 +860,10 @@ const MULTI_QA_MPNET_BASE_DOT_V1_EMBEDDINGS: TextEmbedderModel = { modelPath: `${BASE_URL}-multi-qa-mpnet-base-dot-v1/${NEXT_VERSION_TAG}/xnnpack/multi_qa_mpnet_base_dot_v1_xnnpack_fp32.pte`, tokenizerPath: `${BASE_URL}-multi-qa-mpnet-base-dot-v1/${NEXT_VERSION_TAG}/tokenizer.json`, }; +const MULTI_QA_MPNET_BASE_DOT_V1_VULKAN_INT8: TextEmbedderModel = { + modelPath: `${BASE_URL}-multi-qa-mpnet-base-dot-v1/${NEXT_VERSION_TAG}/vulkan/multi_qa_mpnet_base_dot_v1_vulkan_int8.pte`, + tokenizerPath: `${BASE_URL}-multi-qa-mpnet-base-dot-v1/${NEXT_VERSION_TAG}/tokenizer.json`, +}; const MULTI_QA_MPNET_BASE_DOT_V1_VULKAN_FP16: TextEmbedderModel = { modelPath: `${BASE_URL}-multi-qa-mpnet-base-dot-v1/${NEXT_VERSION_TAG}/vulkan/multi_qa_mpnet_base_dot_v1_vulkan_fp16.pte`, tokenizerPath: `${BASE_URL}-multi-qa-mpnet-base-dot-v1/${NEXT_VERSION_TAG}/tokenizer.json`, @@ -2611,6 +2619,7 @@ export const models = { */ ALL_MPNET_BASE_V2: variants({ XNNPACK_FP32: ALL_MPNET_BASE_V2_EMBEDDINGS, + VULKAN_INT8: ALL_MPNET_BASE_V2_VULKAN_INT8, VULKAN_FP16: ALL_MPNET_BASE_V2_VULKAN_FP16, }), /** @@ -2629,6 +2638,7 @@ export const models = { MULTI_QA_MPNET_BASE_DOT_V1: variants({ XNNPACK_FP32: MULTI_QA_MPNET_BASE_DOT_V1_EMBEDDINGS, VULKAN_FP16: MULTI_QA_MPNET_BASE_DOT_V1_VULKAN_FP16, + VULKAN_INT8: MULTI_QA_MPNET_BASE_DOT_V1_VULKAN_INT8, }), /** * 384-dimensional sentence transformer supporting 50+ languages for From f8241d419b1b792d730bf44631670d12b41cdc7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20S=C5=82uszniak?= Date: Wed, 2 Sep 2026 11:33:06 +0200 Subject: [PATCH 22/24] fix(models): keep fp16 the Android default for both mpnet embedders The int8 export is 6 to 8 times slower than fp16 on a Mali-G76: 25.4 s against 3.0 s per execution at 382 tokens, 5.4 s against 1.17 s at 128. It wins 1.34x on an Adreno 840, so this is not a bad export, it is a kernel that only suits one GPU family. linear_q8csw_tiled unpacks the int8 weights with bitfieldExtract inside the accumulation loop and ships with TILE_M4 = TILE_N4 = TILE_K4 = 1, so each unpacked weight texel feeds only four FMAs. Adreno issues integer and float at comparable rates and still comes out ahead on memory traffic; Bifrost dual-issues fp16 on the FMA pipe but not the integer path, so the fp16 model collects a 2x bonus the int8 model cannot. The registry resolves by backend, not by GPU vendor, so one Android default has to serve both. fp16 is the safe one. VULKAN_INT8 stays reachable by name and is still the better choice on Adreno. --- packages/react-native-executorch/src/models.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts index 8aa102dbf7..00012bb7db 100644 --- a/packages/react-native-executorch/src/models.ts +++ b/packages/react-native-executorch/src/models.ts @@ -2619,8 +2619,8 @@ export const models = { */ ALL_MPNET_BASE_V2: variants({ XNNPACK_FP32: ALL_MPNET_BASE_V2_EMBEDDINGS, - VULKAN_INT8: ALL_MPNET_BASE_V2_VULKAN_INT8, VULKAN_FP16: ALL_MPNET_BASE_V2_VULKAN_FP16, + VULKAN_INT8: ALL_MPNET_BASE_V2_VULKAN_INT8, }), /** * 384-dimensional sentence transformer fine-tuned specifically for semantic From 9af11c0287efb624ec01828e85a1a924fc49099e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20S=C5=82uszniak?= Date: Wed, 2 Sep 2026 20:02:17 +0200 Subject: [PATCH 23/24] refactor(models): address review on default variant resolution Derive BackendTag from ALL_BACKENDS, lowercase the tags so the registered names no longer need matching case, fold the simulator carve-out into BACKEND_ORDER, rename usableBackends to getCandidateBackends, read the backends through the typed utils helper, and inline backendOf. --- .../react-native-executorch/src/models.ts | 61 ++++++++----------- 1 file changed, 27 insertions(+), 34 deletions(-) diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts index 00012bb7db..8ef63f379f 100644 --- a/packages/react-native-executorch/src/models.ts +++ b/packages/react-native-executorch/src/models.ts @@ -15,6 +15,7 @@ import { Platform } from 'react-native'; import { rnexecutorchJsi } from './native/bridge'; +import { getRegisteredBackends } from './utils'; import type { ClassifierModel } from './extensions/cv/tasks/classification'; import type { ObjectDetectorModel } from './extensions/cv/tasks/objectDetection'; import type { StyleTransferModel } from './extensions/cv/tasks/styleTransfer'; @@ -65,60 +66,50 @@ import { // variants are declared in. A group whose best export does not follow from // that order pins one per platform — see the second argument of `variants`. +/** Every backend the registry publishes for, spelled as the variant keys spell it. */ +const ALL_BACKENDS = ['xnnpack', 'coreml', 'mlx', 'vulkan'] as const; + /** The backend prefix a variant key starts with. */ -type BackendTag = 'XNNPACK' | 'COREML' | 'MLX' | 'VULKAN'; +type BackendTag = (typeof ALL_BACKENDS)[number]; /** The platforms the registry resolves defaults for. */ type TargetPlatform = 'ios' | 'android'; -const ALL_BACKENDS: readonly BackendTag[] = ['XNNPACK', 'COREML', 'MLX', 'VULKAN']; +const PLATFORM: TargetPlatform = Platform.OS === 'ios' ? 'ios' : 'android'; // Accelerators lead and XNNPACK trails: a model is only exported to Core ML, // MLX or Vulkan once it has been shown to run better there, and XNNPACK is the // one backend every model exports to. Core ML sits above MLX only to make the // order deterministic; every group publishing both pins its winner explicitly. +// +// The iOS simulator links the Core ML backend but cannot run it: no Neural +// Engine, and MPSGraph refuses the compiled models. MLX only ever ships a +// device slice, so it has nothing to run there either. const BACKEND_ORDER: Record = { - ios: ['COREML', 'MLX', 'XNNPACK'], - android: ['VULKAN', 'XNNPACK'], + ios: rnexecutorchJsi.isEmulator === true ? ['xnnpack'] : ['coreml', 'mlx', 'xnnpack'], + android: ['vulkan', 'xnnpack'], }; -const PLATFORM: TargetPlatform = Platform.OS === 'ios' ? 'ios' : 'android'; - /** - * Backends linked into this binary and usable on this device. - * @returns The usable tags — every one of them when the native runtime cannot - * be asked, so that a missing answer widens the choice rather than emptying it. + * The backends this platform may default to, best first. + * @returns This platform's order, less every backend the binary was not linked + * with — or the order untouched when the native runtime cannot be asked, so + * that a missing answer widens the choice rather than emptying it. */ -function usableBackends(): ReadonlySet { +function getCandidateBackends(): readonly BackendTag[] { let registered: readonly string[] = []; try { - registered = rnexecutorchJsi.getExecuTorchRegisteredBackends(); + registered = getRegisteredBackends(); } catch { registered = []; } - if (registered.length === 0) return new Set(ALL_BACKENDS); + if (registered.length === 0) return BACKEND_ORDER[PLATFORM]; const names = registered.map((name) => name.toLowerCase()); - const usable = ALL_BACKENDS.filter((tag) => - names.some((name) => name.startsWith(tag.toLowerCase())) - ); - - // The simulator links the Core ML backend but cannot run it: no Neural - // Engine, and MPSGraph refuses the compiled models. MLX only ever ships a - // device slice, so it drops out of `registered` on its own. - const onSimulator = PLATFORM === 'ios' && rnexecutorchJsi.isEmulator === true; - return new Set(onSimulator ? usable.filter((tag) => tag !== 'COREML' && tag !== 'MLX') : usable); + return BACKEND_ORDER[PLATFORM].filter((tag) => names.some((name) => name.startsWith(tag))); } -const USABLE_BACKENDS = usableBackends(); - -/** - * Reads the backend out of a variant key. - * @param key The variant key, e.g. `COREML_FP16`. - * @returns The backend the key names, or `undefined` when it names none. - */ -const backendOf = (key: string): BackendTag | undefined => - ALL_BACKENDS.find((tag) => key === tag || key.startsWith(`${tag}_`)); +const CANDIDATE_BACKENDS = getCandidateBackends(); /** * Picks the variant key this platform should default to. @@ -127,11 +118,13 @@ const backendOf = (key: string): BackendTag | undefined => * @returns The chosen key. */ function pickVariant(keys: readonly string[], pin?: string): string { - const pinned = pin === undefined ? undefined : backendOf(pin); - if (pin !== undefined && keys.includes(pin) && pinned && USABLE_BACKENDS.has(pinned)) return pin; + const backendOf = (key: string) => key.toLowerCase().split('_')[0] as BackendTag; + + if (pin !== undefined && keys.includes(pin) && CANDIDATE_BACKENDS.includes(backendOf(pin))) { + return pin; + } - for (const tag of BACKEND_ORDER[PLATFORM]) { - if (!USABLE_BACKENDS.has(tag)) continue; + for (const tag of CANDIDATE_BACKENDS) { const match = keys.find((key) => backendOf(key) === tag); if (match !== undefined) return match; } From 0c1020c04563a8391be88a33332dbe2e9d5fb28d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20S=C5=82uszniak?= Date: Thu, 3 Sep 2026 13:02:15 +0200 Subject: [PATCH 24/24] chore(cspell): allow recompiles and xzmf Both come from download-libs.js on rne-rewrite (#1407) and surface as cspell warnings on this PR's merge check. --- .cspell-wordlist.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.cspell-wordlist.txt b/.cspell-wordlist.txt index 051b033392..77427e9034 100644 --- a/.cspell-wordlist.txt +++ b/.cspell-wordlist.txt @@ -351,3 +351,5 @@ həlˈoʊ NSURL backgrounding LibriSpeech +recompiles +xzmf