diff --git a/.cspell-wordlist.txt b/.cspell-wordlist.txt index 58d46387fd..051b033392 100644 --- a/.cspell-wordlist.txt +++ b/.cspell-wordlist.txt @@ -350,3 +350,4 @@ phonemizes həlˈoʊ NSURL backgrounding +LibriSpeech diff --git a/apps/computer-vision/app/detection/index.tsx b/apps/computer-vision/app/detection/index.tsx index fdee01d0a1..ba5410a077 100644 --- a/apps/computer-vision/app/detection/index.tsx +++ b/apps/computer-vision/app/detection/index.tsx @@ -16,11 +16,11 @@ import { BoundingBox } from '../../components/BoundingBox'; const MODEL_OPTIONS: ModelOption[] = [ { label: 'SSDLite 320 MobileNet V3 Large (XNNPACK FP32)', - value: models.objectDetection.SSDLITE320_MOBILENET_V3_LARGE.DEFAULT, + value: models.objectDetection.SSDLITE320_MOBILENET_V3_LARGE.XNNPACK_FP32, }, { label: 'RF-DETR Nano (XNNPACK FP32)', - value: models.objectDetection.RFDETR_NANO.DEFAULT, + value: models.objectDetection.RFDETR_NANO.XNNPACK_FP32, }, { label: 'RF-DETR Nano (CoreML FP16)', diff --git a/apps/computer-vision/app/keypoint/index.tsx b/apps/computer-vision/app/keypoint/index.tsx index d0649996da..377d502666 100644 --- a/apps/computer-vision/app/keypoint/index.tsx +++ b/apps/computer-vision/app/keypoint/index.tsx @@ -16,7 +16,7 @@ import { BoundingBox } from '../../components/BoundingBox'; const MODEL_OPTIONS: ModelOption[] = [ { label: 'BlazeFace (XNNPACK FP32)', - value: models.keypointDetection.BLAZEFACE.DEFAULT, + value: models.keypointDetection.BLAZEFACE.XNNPACK_FP32, }, { label: 'YOLO26 Pose (XNNPACK FP32)', @@ -27,8 +27,8 @@ const MODEL_OPTIONS: ModelOption[] = [ value: models.keypointDetection.RFDETR_KEYPOINT.XNNPACK_FP32, }, { - label: 'RF-DETR Keypoint (CoreML FP32)', - value: models.keypointDetection.RFDETR_KEYPOINT.COREML_FP32, + label: 'RF-DETR Keypoint (CoreML FP16)', + value: models.keypointDetection.RFDETR_KEYPOINT.COREML_FP16, disabled: Platform.OS !== 'ios', }, { diff --git a/apps/computer-vision/package.json b/apps/computer-vision/package.json index f903f39c4a..26de031de0 100644 --- a/apps/computer-vision/package.json +++ b/apps/computer-vision/package.json @@ -10,8 +10,10 @@ "instanceSegmentation", "keypointDetection", "objectDetection", + "ocr", "semanticSegmentation", - "styleTransfer" + "styleTransfer", + "textToImage" ] }, "scripts": { diff --git a/apps/speech/app/audio-file-transcription/index.tsx b/apps/speech/app/audio-file-transcription/index.tsx index 1b58375c40..1bba02bab5 100644 --- a/apps/speech/app/audio-file-transcription/index.tsx +++ b/apps/speech/app/audio-file-transcription/index.tsx @@ -73,6 +73,31 @@ const MODELS = [ config: models.speechToText.WHISPER.TINY.MLX_BF16, disabled: Platform.OS !== 'ios', }, + { + name: 'Tiny Multilingual (Vulkan fp16)', + config: models.speechToText.WHISPER.TINY.VULKAN_FP16, + disabled: Platform.OS !== 'android', + }, + { + name: 'Tiny Multilingual (Vulkan int8)', + config: models.speechToText.WHISPER.TINY.VULKAN_INT8, + disabled: Platform.OS !== 'android', + }, + { + name: 'Tiny English (Vulkan fp16)', + config: models.speechToText.WHISPER.EN.TINY.VULKAN_FP16, + disabled: Platform.OS !== 'android', + }, + { + name: 'Base Multilingual (Vulkan fp16)', + config: models.speechToText.WHISPER.BASE.VULKAN_FP16, + disabled: Platform.OS !== 'android', + }, + { + name: 'Small Multilingual (Vulkan fp16)', + config: models.speechToText.WHISPER.SMALL.VULKAN_FP16, + disabled: Platform.OS !== 'android', + }, { name: 'Base Multilingual (CPU)', config: models.speechToText.WHISPER.BASE.XNNPACK_FP32, diff --git a/apps/speech/app/microphone-transcription/index.tsx b/apps/speech/app/microphone-transcription/index.tsx index 1355e5991a..b30ec6b0b9 100644 --- a/apps/speech/app/microphone-transcription/index.tsx +++ b/apps/speech/app/microphone-transcription/index.tsx @@ -66,6 +66,31 @@ const MODELS = [ config: models.speechToText.WHISPER.TINY.MLX_BF16, disabled: Platform.OS !== 'ios', }, + { + name: 'Tiny Multilingual (Vulkan fp16)', + config: models.speechToText.WHISPER.TINY.VULKAN_FP16, + disabled: Platform.OS !== 'android', + }, + { + name: 'Tiny Multilingual (Vulkan int8)', + config: models.speechToText.WHISPER.TINY.VULKAN_INT8, + disabled: Platform.OS !== 'android', + }, + { + name: 'Tiny English (Vulkan fp16)', + config: models.speechToText.WHISPER.EN.TINY.VULKAN_FP16, + disabled: Platform.OS !== 'android', + }, + { + name: 'Base Multilingual (Vulkan fp16)', + config: models.speechToText.WHISPER.BASE.VULKAN_FP16, + disabled: Platform.OS !== 'android', + }, + { + name: 'Small Multilingual (Vulkan fp16)', + config: models.speechToText.WHISPER.SMALL.VULKAN_FP16, + disabled: Platform.OS !== 'android', + }, { name: 'Base Multilingual (CPU)', config: models.speechToText.WHISPER.BASE.XNNPACK_FP32, diff --git a/apps/speech/app/text-to-speech/index.tsx b/apps/speech/app/text-to-speech/index.tsx index b6dcb2be1b..1e041a4a24 100644 --- a/apps/speech/app/text-to-speech/index.tsx +++ b/apps/speech/app/text-to-speech/index.tsx @@ -53,11 +53,14 @@ const STEPS_OPTIONS = [ const MODEL_OPTIONS = [ { label: 'XNNPACK (CPU)', value: 'XNNPACK_FP32' as const }, { label: 'MLX (Apple Silicon)', value: 'MLX_FP32' as const, disabled: Platform.OS !== 'ios' }, + { label: 'Vulkan (GPU)', value: 'VULKAN_FP16' as const, disabled: Platform.OS !== 'android' }, ]; function TTSContent() { const [text, setText] = useState(SAMPLE_TEXT); - const [selectedModel, setSelectedModel] = useState<'XNNPACK_FP32' | 'MLX_FP32'>('XNNPACK_FP32'); + const [selectedModel, setSelectedModel] = useState<'XNNPACK_FP32' | 'MLX_FP32' | 'VULKAN_FP16'>( + Platform.OS === 'android' ? 'VULKAN_FP16' : 'XNNPACK_FP32' + ); const [selectedVoice, setSelectedVoice] = useState('F1'); const [selectedLang, setSelectedLang] = useState('en'); const [speed, setSpeed] = useState(1.05); diff --git a/apps/speech/package.json b/apps/speech/package.json index f6dc593c20..a1a13c58b5 100644 --- a/apps/speech/package.json +++ b/apps/speech/package.json @@ -5,6 +5,7 @@ "react-native-executorch": { "features": [ "vad", + "speechToText", "textToSpeech" ] }, diff --git a/docs/docs/01-fundamentals/01-getting-started.md b/docs/docs/01-fundamentals/01-getting-started.md index 3c9e2c5603..0f814c5d7c 100644 --- a/docs/docs/01-fundamentals/01-getting-started.md +++ b/docs/docs/01-fundamentals/01-getting-started.md @@ -213,17 +213,48 @@ The three lists are merged, so you can pair a `features` set with an extra `back ### Feature → backend / lib mapping -These tasks are available today: +Each feature provisions the union of the backends its models are published for, so that the `DEFAULT` variant of every model in that family can resolve to the fastest export the device supports (see below). | Feature | Backends | Extra libs | | --- | --- | --- | +| `llm` | xnnpack, mlx | — | +| `multimodalLLM` | xnnpack, mlx, vulkan | opencv | +| `privacyFilter` | xnnpack, mlx | — | +| `speechToText` | xnnpack, coreml, mlx | — | +| `textToSpeech` | xnnpack, mlx | phonemis | +| `vad` | xnnpack | — | +| `textEmbeddings` | xnnpack, mlx | — | +| `imageEmbeddings` | xnnpack, coreml, mlx | opencv | | `classification` | xnnpack, coreml | opencv | -| `semanticSegmentation` | xnnpack | opencv | -| `styleTransfer` | xnnpack, coreml | opencv | +| `objectDetection` | xnnpack, coreml | opencv | | `keypointDetection` | xnnpack, coreml, mlx | opencv | +| `semanticSegmentation` | xnnpack, coreml | opencv | +| `instanceSegmentation` | xnnpack, coreml | opencv | +| `ocr` | xnnpack, coreml, vulkan | opencv | +| `verticalOCR` | xnnpack | opencv | +| `styleTransfer` | xnnpack, coreml | opencv | +| `textToImage` | xnnpack, coreml | opencv | +| `segmentAnything` | xnnpack, coreml | opencv | | `tokenizer` | — | — | -The map also contains forward-looking entries (`llm`, `multimodalLLM`, `speechToText`, `objectDetection`, `ocr`, …) for tasks that are not yet exposed in the JS API; requesting one provisions the right binaries but has no hook to call yet. +### How the backends you pick change which model runs + +Models published for more than one backend expose their exports as named variants next to a `DEFAULT` alias: + +```ts +models.classification.EFFICIENTNET_V2_S.DEFAULT; // resolved for this device +models.classification.EFFICIENTNET_V2_S.COREML_FP16; // always this export +models.classification.EFFICIENTNET_V2_S.XNNPACK_INT8; +``` + +`DEFAULT` is not a fixed file. It is resolved when the library loads, to the fastest export the device can actually run. A model is only exported to an accelerated backend once it has been shown to run better there, so a published accelerated variant is preferred and XNNPACK is the fallback: + +- **iOS device** — Core ML where one exists, otherwise MLX, otherwise XNNPACK. +- **Android** — Vulkan where one exists, otherwise XNNPACK. +- **iOS simulator** — XNNPACK. The simulator has no Neural Engine, cannot run Core ML models at all, and MLX ships a device slice only. +- **All of them, narrowed by your config** — only backends your app downloaded are considered. Trimming `coreml` out of an iOS build moves those `DEFAULT`s to the next best export rather than failing to load. + +A handful of models publish both a Core ML and an MLX export. There the two are close enough that the winner is a per-model benchmark result, so the registry pins it explicitly rather than letting the order above decide. Naming any variant directly always overrides the resolution. ### Platform notes diff --git a/packages/react-native-executorch/__tests__/api/modelVariants.test.ts b/packages/react-native-executorch/__tests__/api/modelVariants.test.ts new file mode 100644 index 0000000000..93b1e229a0 --- /dev/null +++ b/packages/react-native-executorch/__tests__/api/modelVariants.test.ts @@ -0,0 +1,397 @@ +/** + * How the `DEFAULT` alias in the `models` registry is chosen. + * + * The alias is resolved once, when the registry module is first imported, from + * the platform and the backends the native binary was linked with. Every case + * below therefore reloads the registry behind a `jest.resetModules()` rather + * than reading the copy the test file imported. + */ +import { readFileSync } from 'fs'; +import { join } from 'path'; + +import { Platform } from 'react-native'; + +import { fakeJsi } from '../support/fakeJsi'; + +type Node = Record; + +const isObject = (value: unknown): value is Node => + typeof value === 'object' && value !== null && !Array.isArray(value); + +const isConfig = (value: unknown): value is Node => + isObject(value) && + (typeof value.modelPath === 'string' || + (isObject(value.modelPaths) && + Object.values(value.modelPaths).every((path) => typeof path === 'string'))); + +/** The `.pte` file(s) a config names, as one comparable string. */ +const modelPathsOf = (config: Node): string => + typeof config.modelPath === 'string' + ? config.modelPath + : Object.values(config.modelPaths as Node) + .map(String) + .sort() + .join('|'); + +const isUpperKey = (key: string) => /^[A-Z0-9_]+$/.test(key); + +/** Every group that names a `DEFAULT`, with the dotted path it sits at. */ +function variantGroups(node: unknown, path: string[] = []): { label: string; group: Node }[] { + if (!isObject(node)) return []; + const here = isConfig(node.DEFAULT) ? [{ label: path.join('.'), group: node }] : []; + const nested = Object.entries(node) + .filter(([key, value]) => key !== 'DEFAULT' && isObject(value)) + .flatMap(([key, value]) => variantGroups(value, [...path, key])); + return [...here, ...nested]; +} + +/** The backend variants a group lists directly, by key. */ +const namedVariants = (group: Node): [string, Node][] => + Object.entries(group).filter( + ([key, value]) => key !== 'DEFAULT' && isUpperKey(key) && isConfig(value) + ) as [string, Node][]; + +/** The variant key a group's `DEFAULT` points at, when it points at its own. */ +const defaultKeyOf = (group: Node): string | undefined => + namedVariants(group).find( + ([, value]) => modelPathsOf(value) === modelPathsOf(group.DEFAULT as Node) + )?.[0]; + +const originalOs = Platform.OS; + +/** + * Points `Platform.OS` at a target, on the module instance the next `require` + * will resolve to. Has to run after `jest.resetModules()`, which hands out a + * fresh `react-native` module. + * @param os The platform to pretend to run on. + */ +function setPlatform(os: 'ios' | 'android'): void { + (require('react-native').Platform as { OS: string }).OS = os; +} + +/** + * Reloads the registry as it would resolve on a given device. + * @param options The platform, the linked backends, and whether the device is + * a simulator. + * @returns The freshly resolved `models` registry. + */ +function registryFor(options: { + os: 'ios' | 'android'; + backends?: string[]; + isEmulator?: boolean; +}): Node { + fakeJsi.setRegisteredBackends( + options.backends ?? ['XnnpackBackend', 'CoreMLBackend', 'MLXBackend', 'VulkanBackend'] + ); + fakeJsi.setIsEmulator(options.isEmulator ?? false); + + jest.resetModules(); + setPlatform(options.os); + return require('../../src/models').models as Node; +} + +/** + * Every `variants(...)` call in the registry source, with the variant keys it + * lists and whether it pins one for iOS. + * + * Prettier gives the two call shapes distinct first lines — `variants({` when + * the group takes no pins, `variants(` when it does — which is what this reads. + * @returns One entry per call, in source order. + */ +function variantsCalls(): { name: string; keys: string[]; pinsIos: boolean }[] { + const source = readFileSync(join(__dirname, '../../src/models.ts'), 'utf8').split('\n'); + const calls: { name: string; keys: string[]; pinsIos: boolean }[] = []; + + for (let line = 0; line < source.length; line++) { + const opened = source[line]!.match(/^(\s*)([A-Z][A-Z0-9_]*): variants\((\{?)$/); + if (!opened) continue; + const [, indent, name, inlineBrace] = opened; + + const closer = inlineBrace ? `${indent}}),` : `${indent}),`; + let end = line + 1; + while (end < source.length && source[end] !== closer) end++; + + const body = source.slice(line + 1, end); + const keys = body.flatMap((entry) => entry.match(/^\s*([A-Z][A-Z0-9_]*):/)?.slice(1) ?? []); + calls.push({ name: name!, keys, pinsIos: body.some((entry) => /\bios:/.test(entry)) }); + line = end; + } + + return calls; +} + +/** Every group of the reloaded registry, paired with the key it defaulted to. */ +const defaultsOf = (registry: Node) => + variantGroups(registry).map(({ label, group }) => ({ + label, + key: defaultKeyOf(group), + path: modelPathsOf(group.DEFAULT as Node), + offers: namedVariants(group).map(([key]) => key), + })); + +afterEach(() => { + jest.resetModules(); + setPlatform(originalOs as 'ios' | 'android'); +}); + +describe('DEFAULT variant resolution', () => { + it('finds groups to check', () => { + expect(defaultsOf(registryFor({ os: 'ios' })).length).toBeGreaterThan(100); + }); + + it('defaults to Core ML on iOS wherever a Core ML export exists and nothing is pinned', () => { + // Core ML leads the iOS order, so it wins by default. It does not always + // win on the device: distiluse pins MLX because MLX measured faster there. + // A group that names an explicit iOS pin has been measured, so it is + // exempt; the pin itself is guarded by `pins one` below. + const pinned = new Set( + variantsCalls() + .filter(({ pinsIos }) => pinsIos) + .map(({ name }) => name) + ); + const offenders = defaultsOf(registryFor({ os: 'ios' })) + .filter(({ label }) => !pinned.has(label.split('.').pop()!)) + .filter(({ offers }) => offers.some((key) => key.startsWith('COREML'))) + .filter(({ key }) => key !== undefined && !key.startsWith('COREML')) + .map(({ label, key }) => `${label}: ${key}`); + + expect(offenders).toEqual([]); + }); + + it('never defaults to an iOS-only backend on Android', () => { + const offenders = defaultsOf(registryFor({ os: 'android' })) + .filter(({ path }) => /\/(coreml|mlx)\//.test(path)) + .map(({ label, path }) => `${label}: ${path}`); + + expect(offenders).toEqual([]); + }); + + it('defaults to XNNPACK on Android where no Vulkan export exists', () => { + const offenders = defaultsOf(registryFor({ os: 'android' })) + .filter(({ offers }) => offers.some((key) => key.startsWith('XNNPACK'))) + .filter(({ offers }) => !offers.some((key) => key.startsWith('VULKAN'))) + .filter(({ key }) => key !== undefined && !key.startsWith('XNNPACK')) + .map(({ label, key }) => `${label}: ${key}`); + + expect(offenders).toEqual([]); + }); + + it('prefers MLX over XNNPACK on iOS where no Core ML export exists', () => { + const offenders = defaultsOf(registryFor({ os: 'ios' })) + .filter(({ offers }) => offers.some((key) => key.startsWith('MLX'))) + .filter(({ offers }) => !offers.some((key) => key.startsWith('COREML'))) + .filter(({ key }) => key !== undefined && !key.startsWith('MLX')) + .map(({ label, key }) => `${label}: ${key}`); + + expect(offenders).toEqual([]); + }); + + it('prefers Vulkan over XNNPACK on Android wherever a Vulkan export exists', () => { + const offenders = defaultsOf(registryFor({ os: 'android' })) + .filter(({ offers }) => offers.some((key) => key.startsWith('VULKAN'))) + .filter(({ key }) => key !== undefined && !key.startsWith('VULKAN')) + .map(({ label, key }) => `${label}: ${key}`); + + expect(offenders).toEqual([]); + }); + + it('never picks Vulkan on iOS or an iOS-only backend on Android', () => { + expect( + defaultsOf(registryFor({ os: 'ios' })) + .filter(({ key }) => key?.startsWith('VULKAN')) + .map(({ label, key }) => `${label}: ${key}`) + ).toEqual([]); + + expect( + defaultsOf(registryFor({ os: 'android' })) + .filter(({ key }) => key?.startsWith('COREML') || key?.startsWith('MLX')) + .map(({ label, key }) => `${label}: ${key}`) + ).toEqual([]); + }); + + it('reads every variant group out of the registry source', () => { + // The pin case below is a source-level check, so it passes for free if the + // scanner stops matching the shape Prettier writes. + const calls = variantsCalls(); + expect(calls.length).toBeGreaterThan(100); + expect(calls.filter(({ pinsIos }) => pinsIos).length).toBeGreaterThan(0); + expect(calls.filter(({ keys }) => keys.length === 0)).toEqual([]); + }); + + it('every group offering both Core ML and MLX pins one', () => { + // Core ML sits above MLX in the backend order only to make the resolution + // deterministic, and that ordering is not a benchmark result. Where a model + // publishes both, the winner has to be written down at the call site so the + // choice is reviewable rather than an accident of the enum order. + // + // A pin to the backend the order would have picked anyway is invisible at + // runtime, so this reads the registry source rather than the resolved + // registry. + const offenders = variantsCalls() + .filter(({ keys }) => keys.some((key) => key.startsWith('COREML'))) + .filter(({ keys }) => keys.some((key) => key.startsWith('MLX'))) + .filter(({ pinsIos }) => !pinsIos) + .map(({ name, keys }) => `${name}: ${keys.join(', ')}`); + + expect(offenders).toEqual([]); + }); + + it('falls back to XNNPACK on the iOS simulator, which runs neither Core ML nor MLX', () => { + const offenders = defaultsOf(registryFor({ os: 'ios', isEmulator: true })) + .filter(({ path }) => /\/(coreml|mlx)\//.test(path)) + .map(({ label, path }) => `${label}: ${path}`); + + expect(offenders).toEqual([]); + }); + + it('falls back to XNNPACK on iOS when the app links XNNPACK only', () => { + const offenders = defaultsOf(registryFor({ os: 'ios', backends: ['XnnpackBackend'] })) + .filter(({ key }) => key !== undefined && !key.startsWith('XNNPACK')) + .map(({ label, key }) => `${label}: ${key}`); + + expect(offenders).toEqual([]); + }); + + it('picks a variant the group actually offers, on every device', () => { + const devices = [ + { os: 'ios' as const }, + { os: 'ios' as const, isEmulator: true }, + { os: 'ios' as const, backends: ['XnnpackBackend'] }, + { os: 'android' as const }, + // An app that opted out of every backend its models were published for: + // the registry still has to name a model rather than yield `undefined`. + { os: 'android' as const, backends: ['CoreMLBackend'] }, + ]; + + for (const device of devices) { + const offenders = variantGroups(registryFor(device)) + .filter(({ group }) => namedVariants(group).length > 0) + .filter(({ group }) => defaultKeyOf(group) === undefined) + .map(({ label }) => `${JSON.stringify(device)} ${label}`); + + expect(offenders).toEqual([]); + } + }); + + it('keeps the same model files reachable through named variants on both platforms', () => { + // Only the default moves per platform; the catalogue itself must not. + const keysOf = (registry: Node) => + defaultsOf(registry) + .map(({ label, offers }) => `${label}: ${offers.join(',')}`) + .sort(); + + expect(keysOf(registryFor({ os: 'ios' }))).toEqual(keysOf(registryFor({ os: 'android' }))); + }); + + it('resolves a family to the default of its first sub-group', () => { + const registry = registryFor({ os: 'ios' }); + const objectDetection = registry.objectDetection as Node; + const yolo = objectDetection.YOLO26 as Node; + const nano = yolo.NANO as Node; + + expect(yolo.DEFAULT).toBe(nano.DEFAULT); + expect(nano.DEFAULT).toBe((nano.SIZE_384 as Node).DEFAULT); + }); +}); + +describe('variant selection rules', () => { + /** The distiluse group, the one place a pin overrides the backend order. */ + const distiluse = (registry: Node): Node => + (registry.textEmbeddings as Node).DISTILUSE_BASE_MULTILINGUAL_CASED_V2 as Node; + + it('breaks a tie within one backend by declaration order', () => { + // With XNNPACK the only linked backend, every group has to land on the + // first XNNPACK variant it declares, whatever else it publishes. + const offenders = defaultsOf(registryFor({ os: 'ios', backends: ['XnnpackBackend'] })) + .filter(({ offers }) => offers.some((key) => key.startsWith('XNNPACK'))) + .filter(({ key, offers }) => key !== offers.find((entry) => entry.startsWith('XNNPACK'))) + .map(({ label, key, offers }) => `${label}: ${key} of ${offers.join(', ')}`); + + expect(offenders).toEqual([]); + }); + + it('honours a pinned variant over the backend order', () => { + // Core ML leads the iOS order, so MLX here is the pin and nothing else. + expect(defaultKeyOf(distiluse(registryFor({ os: 'ios' })))).toBe('MLX_INT8'); + }); + + it('ignores a pin whose backend the app did not link in', () => { + const registry = registryFor({ os: 'ios', backends: ['XnnpackBackend'] }); + expect(defaultKeyOf(distiluse(registry))).toBe('XNNPACK_8DA4W'); + }); + + it('applies a pin only on the platform it names', () => { + // The same group pins MLX for iOS only; Android resolves by its own order. + expect(defaultKeyOf(distiluse(registryFor({ os: 'android' })))).toBe('VULKAN_FP16'); + }); +}); + +describe('feature map', () => { + // `models...DEFAULT` only reaches the accelerated export when + // the app downloaded that backend, and `features` is the documented way to + // say which backends an app needs. A family whose feature entry is missing a + // backend it publishes therefore falls back to XNNPACK forever, quietly. + // + // Categories map to feature names one-to-one except where noted; a category + // added without an entry here fails the coverage case below. + const FEATURE_OF_CATEGORY: Record = { + classification: 'classification', + styleTransfer: 'styleTransfer', + semanticSegmentation: 'semanticSegmentation', + objectDetection: 'objectDetection', + keypointDetection: 'keypointDetection', + instanceSegmentation: 'instanceSegmentation', + voiceActivityDetection: 'vad', + speechToText: 'speechToText', + tokenizer: 'tokenizer', + llm: 'llm', + textEmbeddings: 'textEmbeddings', + privacyFilter: 'privacyFilter', + imageEmbeddings: 'imageEmbeddings', + textToImage: 'textToImage', + textToSpeech: 'textToSpeech', + ocr: 'ocr', + }; + + const { FEATURE_MAP } = require('../../scripts/download-libs.js'); + + /** Every backend folder the URLs under a registry category point into. */ + function publishedBackends(node: unknown): Set { + const found = new Set(); + const walk = (value: unknown): void => { + if (typeof value === 'string') { + const match = value.match(/\/(xnnpack|coreml|mlx|vulkan)\//); + if (match) found.add(match[1]!); + } else if (Array.isArray(value)) value.forEach(walk); + else if (isObject(value)) Object.values(value).forEach(walk); + }; + walk(node); + return found; + } + + const registry = registryFor({ os: 'ios' }); + + it('names a feature for every registry category', () => { + expect(Object.keys(registry).filter((category) => !FEATURE_OF_CATEGORY[category])).toEqual([]); + }); + + it('provisions every backend the registry publishes for that feature', () => { + const offenders: string[] = []; + + for (const [category, node] of Object.entries(registry)) { + const feature = FEATURE_OF_CATEGORY[category]!; + // Multimodal LLMs are split into their own feature; both entries cover + // the `llm` category, so the union of the two is what an LLM app gets. + const provisioned = new Set([ + ...FEATURE_MAP[feature].backends, + ...(feature === 'llm' ? FEATURE_MAP.multimodalLLM.backends : []), + ]); + + for (const backend of publishedBackends(node)) { + if (!provisioned.has(backend)) offenders.push(`${feature} is missing ${backend}`); + } + } + + expect(offenders.sort()).toEqual([]); + }); +}); diff --git a/packages/react-native-executorch/scripts/download-libs.js b/packages/react-native-executorch/scripts/download-libs.js index 828ee6f6e0..1ae5d46af4 100644 --- a/packages/react-native-executorch/scripts/download-libs.js +++ b/packages/react-native-executorch/scripts/download-libs.js @@ -115,24 +115,37 @@ const ALL_LIBS = ['opencv', 'phonemis']; // features -> { backends, libs } // Backend lists are the union of what at least one model in that family ships // today (per src/models.ts). When a new variant lands for a model that adds -// e.g. coreml or vulkan support, bump the family here. +// e.g. coreml or vulkan support, bump the family here. Leaving one out is not +// only a missed optimization: `models...DEFAULT` resolves to the +// fastest export the device can actually run, so a backend that never gets +// downloaded silently drops that model back to XNNPACK. The registry test +// `feature map` in __tests__/api/modelVariants.test.ts holds this in sync. const FEATURE_MAP = { - // Text-only LLMs ship xnnpack + mlx (Gemma 4 ships an MLX iOS export). - llm: { backends: ['xnnpack', 'mlx'], libs: [] }, + // Text-only LLMs ship xnnpack + mlx (Gemma 4 ships an MLX iOS export) and + // vulkan (Gemma 4 E2B ships a Vulkan export). + llm: { backends: ['xnnpack', 'mlx', 'vulkan'], libs: [] }, // Multimodal LLMs add vulkan (Gemma-3-multimodal ships a Vulkan export) and // mlx (Gemma 4 ships an MLX iOS export); the vision encoder needs opencv. multimodalLLM: { backends: ['xnnpack', 'mlx', 'vulkan'], libs: ['opencv'] }, // Privacy filter classifiers ship xnnpack + an MLX iOS export. privacyFilter: { backends: ['xnnpack', 'mlx'], libs: [] }, - // Whisper ships xnnpack + coreml. - speechToText: { backends: ['xnnpack', 'coreml'], libs: [] }, - // Kokoro ships xnnpack + coreml. - textToSpeech: { backends: ['xnnpack', 'coreml'], libs: ['phonemis'] }, + // Whisper ships xnnpack, coreml, an MLX iOS export and a Vulkan Android one. + speechToText: { backends: ['xnnpack', 'coreml', 'mlx', 'vulkan'], libs: [] }, + // Kokoro ships xnnpack + coreml; Supertonic adds an MLX iOS export and a + // Vulkan Android one. + textToSpeech: { + backends: ['xnnpack', 'coreml', 'mlx', 'vulkan'], + libs: ['phonemis'], + }, // FSMN VAD — xnnpack only. vad: { backends: ['xnnpack'], libs: [] }, - // LFM2.5-Embedding ships an MLX iOS export alongside xnnpack. - textEmbeddings: { backends: ['xnnpack', 'mlx'], libs: [] }, - imageEmbeddings: { backends: ['xnnpack'], libs: ['opencv'] }, + // The MiniLM/CLIP-text/distiluse family ships coreml alongside xnnpack, + // LFM2.5-Embedding and distiluse add MLX iOS exports, and every sentence + // transformer now ships a vulkan Android export. + textEmbeddings: { backends: ['xnnpack', 'coreml', 'mlx', 'vulkan'], libs: [] }, + // CLIP's vision encoder ships xnnpack, coreml and mlx, plus a vulkan Android + // export. + imageEmbeddings: { backends: ['xnnpack', 'coreml', 'mlx', 'vulkan'], libs: ['opencv'] }, // EfficientNet ships xnnpack + coreml. classification: { backends: ['xnnpack', 'coreml'], libs: ['opencv'] }, // YOLO is xnnpack-only, ssdlite/rf_detr add coreml → union. @@ -141,8 +154,8 @@ const FEATURE_MAP = { // keypoint adds coreml + mlx → union. (Named to track the useKeypointDetector // hook; main calls this poseEstimation.) keypointDetection: { backends: ['xnnpack', 'coreml', 'mlx'], libs: ['opencv'] }, - // DeepLab/FCN/LR-ASPP/selfie — xnnpack only. - semanticSegmentation: { backends: ['xnnpack'], libs: ['opencv'] }, + // DeepLab/FCN/LR-ASPP/selfie all ship xnnpack + coreml. + semanticSegmentation: { backends: ['xnnpack', 'coreml'], libs: ['opencv'] }, // YOLO-seg xnnpack-only, rf_detr-seg/fastsam add coreml → union. instanceSegmentation: { backends: ['xnnpack', 'coreml'], libs: ['opencv'] }, // PP-OCRv6 (DBNet + SVTR) ships xnnpack, coreml and vulkan → union. @@ -150,8 +163,8 @@ const FEATURE_MAP = { verticalOCR: { backends: ['xnnpack'], libs: ['opencv'] }, // All style-transfer presets ship xnnpack + coreml. styleTransfer: { backends: ['xnnpack', 'coreml'], libs: ['opencv'] }, - // BK-SDM — xnnpack only. - textToImage: { backends: ['xnnpack'], libs: ['opencv'] }, + // SDXS ships xnnpack + coreml. + textToImage: { backends: ['xnnpack', 'coreml'], libs: ['opencv'] }, // FastSAM ships xnnpack + coreml. segmentAnything: { backends: ['xnnpack', 'coreml'], libs: ['opencv'] }, // Tokenizer is pure-CPU string ops resolved from libexecutorch; needs no @@ -457,8 +470,12 @@ async function main() { console.log('[react-native-executorch] Native libs ready.'); } -main().catch((err) => { - console.error('[react-native-executorch] Failed to download native libs:', err.message); - console.error(' You can set RNET_SKIP_DOWNLOAD=1 to skip and provide libs manually.'); - process.exit(1); -}); +if (require.main === module) { + main().catch((err) => { + console.error('[react-native-executorch] Failed to download native libs:', err.message); + console.error(' You can set RNET_SKIP_DOWNLOAD=1 to skip and provide libs manually.'); + process.exit(1); + }); +} + +module.exports = { ALL_BACKENDS, ALL_LIBS, FEATURE_MAP }; diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts index eb47fd981a..8ef63f379f 100644 --- a/packages/react-native-executorch/src/models.ts +++ b/packages/react-native-executorch/src/models.ts @@ -5,8 +5,17 @@ * vision, speech synthesis/recognition, natural language processing, and large * language models (LLMs). Each entry includes verified remote `.pte` download * URLs, tokenizer/phonemizer files, preprocessing parameters, and label maps. + * + * A model that ships several exports lists them under backend-tagged keys and + * wraps the group in `variants`, which adds the `DEFAULT` alias resolving to + * the fastest export the current platform can run. Within one backend the + * first variant declared wins, so keep the group ordered best-first. */ +import { Platform } from 'react-native'; + +import { rnexecutorchJsi } from './native/bridge'; +import { getRegisteredBackends } from './utils'; import type { ClassifierModel } from './extensions/cv/tasks/classification'; import type { ObjectDetectorModel } from './extensions/cv/tasks/objectDetection'; import type { StyleTransferModel } from './extensions/cv/tasks/styleTransfer'; @@ -49,6 +58,119 @@ import { type SupertonicDefaultVoiceName, } from './constants'; +// ============================================================================= +// DEFAULT variant resolution +// ============================================================================= +// `DEFAULT` is resolved once, when this module is first imported, from the +// platform, the backends the binary was linked with, and the order the +// variants are declared in. A group whose best export does not follow from +// that order pins one per platform — see the second argument of `variants`. + +/** Every backend the registry publishes for, spelled as the variant keys spell it. */ +const ALL_BACKENDS = ['xnnpack', 'coreml', 'mlx', 'vulkan'] as const; + +/** The backend prefix a variant key starts with. */ +type BackendTag = (typeof ALL_BACKENDS)[number]; + +/** The platforms the registry resolves defaults for. */ +type TargetPlatform = 'ios' | 'android'; + +const PLATFORM: TargetPlatform = Platform.OS === 'ios' ? 'ios' : 'android'; + +// Accelerators lead and XNNPACK trails: a model is only exported to Core ML, +// MLX or Vulkan once it has been shown to run better there, and XNNPACK is the +// one backend every model exports to. Core ML sits above MLX only to make the +// order deterministic; every group publishing both pins its winner explicitly. +// +// The iOS simulator links the Core ML backend but cannot run it: no Neural +// Engine, and MPSGraph refuses the compiled models. MLX only ever ships a +// device slice, so it has nothing to run there either. +const BACKEND_ORDER: Record = { + ios: rnexecutorchJsi.isEmulator === true ? ['xnnpack'] : ['coreml', 'mlx', 'xnnpack'], + android: ['vulkan', 'xnnpack'], +}; + +/** + * The backends this platform may default to, best first. + * @returns This platform's order, less every backend the binary was not linked + * with — or the order untouched when the native runtime cannot be asked, so + * that a missing answer widens the choice rather than emptying it. + */ +function getCandidateBackends(): readonly BackendTag[] { + let registered: readonly string[] = []; + try { + registered = getRegisteredBackends(); + } catch { + registered = []; + } + if (registered.length === 0) return BACKEND_ORDER[PLATFORM]; + + const names = registered.map((name) => name.toLowerCase()); + return BACKEND_ORDER[PLATFORM].filter((tag) => names.some((name) => name.startsWith(tag))); +} + +const CANDIDATE_BACKENDS = getCandidateBackends(); + +/** + * Picks the variant key this platform should default to. + * @param keys The group's variant keys, in declaration order. + * @param pin The key pinned for this platform, if any. + * @returns The chosen key. + */ +function pickVariant(keys: readonly string[], pin?: string): string { + const backendOf = (key: string) => key.toLowerCase().split('_')[0] as BackendTag; + + if (pin !== undefined && keys.includes(pin) && CANDIDATE_BACKENDS.includes(backendOf(pin))) { + return pin; + } + + for (const tag of CANDIDATE_BACKENDS) { + const match = keys.find((key) => backendOf(key) === tag); + if (match !== undefined) return match; + } + + // No preferred backend is both published for this model and linked into the + // build — an app that opted out of the backends its models need. Hand back + // the first variant so the registry still names a model and the failure + // surfaces at load, where the error says which backend is missing. + return keys[0]!; +} + +/** + * Adds a platform-resolved `DEFAULT` to a group of backend variants. + * + * Declare the variants best-first within each backend: with several exports + * from the same backend, the earliest one wins. + * @typeParam V The variant map. + * @param map The group's variants, keyed by backend and precision. + * @param pinned Variant keys to prefer on a given platform, for groups whose + * best export does not follow from the declaration order. Ignored when the + * pinned variant's backend is not linked into the build. + * @returns The variants, plus the `DEFAULT` alias for this platform. + */ +function variants>( + map: V, + pinned?: Partial>> +): V & { readonly DEFAULT: V[keyof V] } { + const key = pickVariant(Object.keys(map), pinned?.[PLATFORM]); + return { ...map, DEFAULT: map[key] as V[keyof V] }; +} + +/** + * Adds a `DEFAULT` to a group of sub-groups — a model family split by scale or + * input size — mirroring the `DEFAULT` of the first sub-group declared, which + * resolved itself per platform. + * @typeParam V The sub-group map. + * @param map The family's sub-groups, most representative first. + * @returns The sub-groups, plus the inherited `DEFAULT`. + */ +function family>( + map: V +): V & { readonly DEFAULT: V[keyof V]['DEFAULT'] } { + const first = Object.keys(map)[0]!; + return { ...map, DEFAULT: map[first]!.DEFAULT as V[keyof V]['DEFAULT'] }; +} + const BASE_URL = 'https://huggingface.co/software-mansion/react-native-executorch'; const VERSION_TAG = 'resolve/v0.9.0'; const NEXT_VERSION_TAG = 'resolve/v0.10.0'; @@ -288,7 +410,7 @@ const RFDETR_NANO_DETECTOR_OPTS = { defaultIouThreshold: 0.55, }; const RFDETR_NANO_DETECTOR_XNNPACK_FP32: ObjectDetectorModel<'xyxy', CocoClass> = { - modelPath: `${BASE_URL}-rfdetr-nano-detector/${VERSION_TAG}/xnnpack/rfdetr_nano_xnnpack_fp32.pte`, + modelPath: `${BASE_URL}-rfdetr-nano-detector/${NEXT_VERSION_TAG}/xnnpack/rfdetr_nano_xnnpack_fp32.pte`, modelOpts: RFDETR_NANO_DETECTOR_OPTS, }; const RFDETR_NANO_DETECTOR_COREML_FP16: ObjectDetectorModel<'xyxy', CocoClass> = { @@ -460,14 +582,26 @@ const YOLO26_POSE_384_XNNPACK_FP32: KeypointDetectorModel<'xyxy', CocoLandmark> modelPath: `${BASE_URL}-yolo26-pose/${NEXT_VERSION_TAG}/xnnpack/yolo26n_pose_384_xnnpack_fp32.pte`, modelOpts: YOLO26_POSE_OPTS, }; +const YOLO26_POSE_384_COREML_FP16: KeypointDetectorModel<'xyxy', CocoLandmark> = { + modelPath: `${BASE_URL}-yolo26-pose/${NEXT_VERSION_TAG}/coreml/yolo26n_pose_384_coreml_fp16.pte`, + modelOpts: YOLO26_POSE_OPTS, +}; const YOLO26_POSE_512_XNNPACK_FP32: KeypointDetectorModel<'xyxy', CocoLandmark> = { modelPath: `${BASE_URL}-yolo26-pose/${NEXT_VERSION_TAG}/xnnpack/yolo26n_pose_512_xnnpack_fp32.pte`, modelOpts: YOLO26_POSE_OPTS, }; +const YOLO26_POSE_512_COREML_FP16: KeypointDetectorModel<'xyxy', CocoLandmark> = { + modelPath: `${BASE_URL}-yolo26-pose/${NEXT_VERSION_TAG}/coreml/yolo26n_pose_512_coreml_fp16.pte`, + modelOpts: YOLO26_POSE_OPTS, +}; const YOLO26_POSE_640_XNNPACK_FP32: KeypointDetectorModel<'xyxy', CocoLandmark> = { modelPath: `${BASE_URL}-yolo26-pose/${NEXT_VERSION_TAG}/xnnpack/yolo26n_pose_640_xnnpack_fp32.pte`, modelOpts: YOLO26_POSE_OPTS, }; +const YOLO26_POSE_640_COREML_FP16: KeypointDetectorModel<'xyxy', CocoLandmark> = { + modelPath: `${BASE_URL}-yolo26-pose/${NEXT_VERSION_TAG}/coreml/yolo26n_pose_640_coreml_fp16.pte`, + modelOpts: YOLO26_POSE_OPTS, +}; const RFDETR_KEYPOINT_OPTS = { boxFormat: 'xyxy' as const, @@ -479,15 +613,15 @@ const RFDETR_KEYPOINT_OPTS = { landmarks: COCO_LANDMARKS, }; const RFDETR_KEYPOINT_XNNPACK_FP32: KeypointDetectorModel<'xyxy', CocoLandmark> = { - modelPath: `${BASE_URL}-rfdetr-keypoint/${VERSION_TAG}/preview/xnnpack/rfdetr_keypoint_preview_xnnpack_fp32.pte`, + modelPath: `${BASE_URL}-rfdetr-keypoint/${NEXT_VERSION_TAG}/preview/xnnpack/rfdetr_keypoint_preview_xnnpack_fp32.pte`, modelOpts: RFDETR_KEYPOINT_OPTS, }; -const RFDETR_KEYPOINT_COREML_FP32: KeypointDetectorModel<'xyxy', CocoLandmark> = { - modelPath: `${BASE_URL}-rfdetr-keypoint/${VERSION_TAG}/preview/coreml/rfdetr_keypoint_preview_coreml_fp32.pte`, +const RFDETR_KEYPOINT_COREML_FP16: KeypointDetectorModel<'xyxy', CocoLandmark> = { + modelPath: `${BASE_URL}-rfdetr-keypoint/${NEXT_VERSION_TAG}/preview/coreml/rfdetr_keypoint_preview_coreml_fp16.pte`, modelOpts: RFDETR_KEYPOINT_OPTS, }; const RFDETR_KEYPOINT_MLX_FP32: KeypointDetectorModel<'xyxy', CocoLandmark> = { - modelPath: `${BASE_URL}-rfdetr-keypoint/${VERSION_TAG}/preview/mlx/rfdetr_keypoint_preview_mlx_fp32.pte`, + modelPath: `${BASE_URL}-rfdetr-keypoint/${NEXT_VERSION_TAG}/preview/mlx/rfdetr_keypoint_preview_mlx_fp32.pte`, modelOpts: RFDETR_KEYPOINT_OPTS, }; @@ -555,66 +689,126 @@ const YOLO26_NANO_SEG_384_XNNPACK_FP32: InstanceSegmenterModel<'xyxy', CocoClass modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/n/xnnpack/yolo26_seg_n_384_xnnpack_fp32.pte`, modelOpts: YOLO26_SEG_OPTS, }; +const YOLO26_NANO_SEG_384_COREML_FP16: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { + modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/n/coreml/yolo26_seg_n_384_coreml_fp16.pte`, + modelOpts: YOLO26_SEG_OPTS, +}; const YOLO26_NANO_SEG_512_XNNPACK_FP32: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/n/xnnpack/yolo26_seg_n_512_xnnpack_fp32.pte`, modelOpts: YOLO26_SEG_OPTS, }; +const YOLO26_NANO_SEG_512_COREML_FP16: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { + modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/n/coreml/yolo26_seg_n_512_coreml_fp16.pte`, + modelOpts: YOLO26_SEG_OPTS, +}; const YOLO26_NANO_SEG_640_XNNPACK_FP32: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/n/xnnpack/yolo26_seg_n_640_xnnpack_fp32.pte`, modelOpts: YOLO26_SEG_OPTS, }; +const YOLO26_NANO_SEG_640_COREML_FP16: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { + modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/n/coreml/yolo26_seg_n_640_coreml_fp16.pte`, + modelOpts: YOLO26_SEG_OPTS, +}; const YOLO26_SMALL_SEG_384_XNNPACK_FP32: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/s/xnnpack/yolo26_seg_s_384_xnnpack_fp32.pte`, modelOpts: YOLO26_SEG_OPTS, }; +const YOLO26_SMALL_SEG_384_COREML_FP16: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { + modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/s/coreml/yolo26_seg_s_384_coreml_fp16.pte`, + modelOpts: YOLO26_SEG_OPTS, +}; const YOLO26_SMALL_SEG_512_XNNPACK_FP32: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/s/xnnpack/yolo26_seg_s_512_xnnpack_fp32.pte`, modelOpts: YOLO26_SEG_OPTS, }; +const YOLO26_SMALL_SEG_512_COREML_FP16: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { + modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/s/coreml/yolo26_seg_s_512_coreml_fp16.pte`, + modelOpts: YOLO26_SEG_OPTS, +}; const YOLO26_SMALL_SEG_640_XNNPACK_FP32: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/s/xnnpack/yolo26_seg_s_640_xnnpack_fp32.pte`, modelOpts: YOLO26_SEG_OPTS, }; +const YOLO26_SMALL_SEG_640_COREML_FP16: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { + modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/s/coreml/yolo26_seg_s_640_coreml_fp16.pte`, + modelOpts: YOLO26_SEG_OPTS, +}; const YOLO26_MEDIUM_SEG_384_XNNPACK_FP32: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/m/xnnpack/yolo26_seg_m_384_xnnpack_fp32.pte`, modelOpts: YOLO26_SEG_OPTS, }; +const YOLO26_MEDIUM_SEG_384_COREML_FP16: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { + modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/m/coreml/yolo26_seg_m_384_coreml_fp16.pte`, + modelOpts: YOLO26_SEG_OPTS, +}; const YOLO26_MEDIUM_SEG_512_XNNPACK_FP32: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/m/xnnpack/yolo26_seg_m_512_xnnpack_fp32.pte`, modelOpts: YOLO26_SEG_OPTS, }; +const YOLO26_MEDIUM_SEG_512_COREML_FP16: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { + modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/m/coreml/yolo26_seg_m_512_coreml_fp16.pte`, + modelOpts: YOLO26_SEG_OPTS, +}; const YOLO26_MEDIUM_SEG_640_XNNPACK_FP32: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/m/xnnpack/yolo26_seg_m_640_xnnpack_fp32.pte`, modelOpts: YOLO26_SEG_OPTS, }; +const YOLO26_MEDIUM_SEG_640_COREML_FP16: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { + modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/m/coreml/yolo26_seg_m_640_coreml_fp16.pte`, + modelOpts: YOLO26_SEG_OPTS, +}; const YOLO26_LARGE_SEG_384_XNNPACK_FP32: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/l/xnnpack/yolo26_seg_l_384_xnnpack_fp32.pte`, modelOpts: YOLO26_SEG_OPTS, }; +const YOLO26_LARGE_SEG_384_COREML_FP16: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { + modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/l/coreml/yolo26_seg_l_384_coreml_fp16.pte`, + modelOpts: YOLO26_SEG_OPTS, +}; const YOLO26_LARGE_SEG_512_XNNPACK_FP32: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/l/xnnpack/yolo26_seg_l_512_xnnpack_fp32.pte`, modelOpts: YOLO26_SEG_OPTS, }; +const YOLO26_LARGE_SEG_512_COREML_FP16: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { + modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/l/coreml/yolo26_seg_l_512_coreml_fp16.pte`, + modelOpts: YOLO26_SEG_OPTS, +}; const YOLO26_LARGE_SEG_640_XNNPACK_FP32: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/l/xnnpack/yolo26_seg_l_640_xnnpack_fp32.pte`, modelOpts: YOLO26_SEG_OPTS, }; +const YOLO26_LARGE_SEG_640_COREML_FP16: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { + modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/l/coreml/yolo26_seg_l_640_coreml_fp16.pte`, + modelOpts: YOLO26_SEG_OPTS, +}; const YOLO26_XLARGE_SEG_384_XNNPACK_FP32: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/x/xnnpack/yolo26_seg_x_384_xnnpack_fp32.pte`, modelOpts: YOLO26_SEG_OPTS, }; +const YOLO26_XLARGE_SEG_384_COREML_FP16: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { + modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/x/coreml/yolo26_seg_x_384_coreml_fp16.pte`, + modelOpts: YOLO26_SEG_OPTS, +}; const YOLO26_XLARGE_SEG_512_XNNPACK_FP32: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/x/xnnpack/yolo26_seg_x_512_xnnpack_fp32.pte`, modelOpts: YOLO26_SEG_OPTS, }; +const YOLO26_XLARGE_SEG_512_COREML_FP16: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { + modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/x/coreml/yolo26_seg_x_512_coreml_fp16.pte`, + modelOpts: YOLO26_SEG_OPTS, +}; const YOLO26_XLARGE_SEG_640_XNNPACK_FP32: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/x/xnnpack/yolo26_seg_x_640_xnnpack_fp32.pte`, modelOpts: YOLO26_SEG_OPTS, }; +const YOLO26_XLARGE_SEG_640_COREML_FP16: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { + modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/x/coreml/yolo26_seg_x_640_coreml_fp16.pte`, + modelOpts: YOLO26_SEG_OPTS, +}; // ============================================================================= // Text Embeddings @@ -623,34 +817,98 @@ const ALL_MINILM_L6_V2_EMBEDDINGS: TextEmbedderModel = { modelPath: `${BASE_URL}-all-MiniLM-L6-v2/${NEXT_VERSION_TAG}/xnnpack/all_minilm_l6_v2_xnnpack_fp32.pte`, tokenizerPath: `${BASE_URL}-all-MiniLM-L6-v2/${NEXT_VERSION_TAG}/tokenizer.json`, }; +const ALL_MINILM_L6_V2_COREML_FP16: TextEmbedderModel = { + modelPath: `${BASE_URL}-all-MiniLM-L6-v2/${NEXT_VERSION_TAG}/coreml/all_minilm_l6_v2_coreml_fp16.pte`, + tokenizerPath: `${BASE_URL}-all-MiniLM-L6-v2/${NEXT_VERSION_TAG}/tokenizer.json`, +}; +const ALL_MINILM_L6_V2_VULKAN_FP16: TextEmbedderModel = { + modelPath: `${BASE_URL}-all-MiniLM-L6-v2/${NEXT_VERSION_TAG}/vulkan/all_minilm_l6_v2_vulkan_fp16.pte`, + tokenizerPath: `${BASE_URL}-all-MiniLM-L6-v2/${NEXT_VERSION_TAG}/tokenizer.json`, +}; const ALL_MPNET_BASE_V2_EMBEDDINGS: TextEmbedderModel = { modelPath: `${BASE_URL}-all-mpnet-base-v2/${NEXT_VERSION_TAG}/xnnpack/all_mpnet_base_v2_xnnpack_fp32.pte`, tokenizerPath: `${BASE_URL}-all-mpnet-base-v2/${NEXT_VERSION_TAG}/tokenizer.json`, }; +const ALL_MPNET_BASE_V2_VULKAN_INT8: TextEmbedderModel = { + modelPath: `${BASE_URL}-all-mpnet-base-v2/${NEXT_VERSION_TAG}/vulkan/all_mpnet_base_v2_vulkan_int8.pte`, + tokenizerPath: `${BASE_URL}-all-mpnet-base-v2/${NEXT_VERSION_TAG}/tokenizer.json`, +}; +const ALL_MPNET_BASE_V2_VULKAN_FP16: TextEmbedderModel = { + modelPath: `${BASE_URL}-all-mpnet-base-v2/${NEXT_VERSION_TAG}/vulkan/all_mpnet_base_v2_vulkan_fp16.pte`, + tokenizerPath: `${BASE_URL}-all-mpnet-base-v2/${NEXT_VERSION_TAG}/tokenizer.json`, +}; const MULTI_QA_MINILM_L6_COS_V1_EMBEDDINGS: TextEmbedderModel = { modelPath: `${BASE_URL}-multi-qa-MiniLM-L6-cos-v1/${NEXT_VERSION_TAG}/xnnpack/multi_qa_minilm_l6_cos_v1_xnnpack_fp32.pte`, tokenizerPath: `${BASE_URL}-multi-qa-MiniLM-L6-cos-v1/${NEXT_VERSION_TAG}/tokenizer.json`, }; +const MULTI_QA_MINILM_L6_COS_V1_COREML_FP16: TextEmbedderModel = { + modelPath: `${BASE_URL}-multi-qa-MiniLM-L6-cos-v1/${NEXT_VERSION_TAG}/coreml/multi_qa_minilm_l6_cos_v1_coreml_fp16.pte`, + tokenizerPath: `${BASE_URL}-multi-qa-MiniLM-L6-cos-v1/${NEXT_VERSION_TAG}/tokenizer.json`, +}; +const MULTI_QA_MINILM_L6_COS_V1_VULKAN_FP16: TextEmbedderModel = { + modelPath: `${BASE_URL}-multi-qa-MiniLM-L6-cos-v1/${NEXT_VERSION_TAG}/vulkan/multi_qa_minilm_l6_cos_v1_vulkan_fp16.pte`, + tokenizerPath: `${BASE_URL}-multi-qa-MiniLM-L6-cos-v1/${NEXT_VERSION_TAG}/tokenizer.json`, +}; const MULTI_QA_MPNET_BASE_DOT_V1_EMBEDDINGS: TextEmbedderModel = { modelPath: `${BASE_URL}-multi-qa-mpnet-base-dot-v1/${NEXT_VERSION_TAG}/xnnpack/multi_qa_mpnet_base_dot_v1_xnnpack_fp32.pte`, tokenizerPath: `${BASE_URL}-multi-qa-mpnet-base-dot-v1/${NEXT_VERSION_TAG}/tokenizer.json`, }; +const MULTI_QA_MPNET_BASE_DOT_V1_VULKAN_INT8: TextEmbedderModel = { + modelPath: `${BASE_URL}-multi-qa-mpnet-base-dot-v1/${NEXT_VERSION_TAG}/vulkan/multi_qa_mpnet_base_dot_v1_vulkan_int8.pte`, + tokenizerPath: `${BASE_URL}-multi-qa-mpnet-base-dot-v1/${NEXT_VERSION_TAG}/tokenizer.json`, +}; +const MULTI_QA_MPNET_BASE_DOT_V1_VULKAN_FP16: TextEmbedderModel = { + modelPath: `${BASE_URL}-multi-qa-mpnet-base-dot-v1/${NEXT_VERSION_TAG}/vulkan/multi_qa_mpnet_base_dot_v1_vulkan_fp16.pte`, + tokenizerPath: `${BASE_URL}-multi-qa-mpnet-base-dot-v1/${NEXT_VERSION_TAG}/tokenizer.json`, +}; const PARAPHRASE_MULTILINGUAL_MINILM_L12_V2_EMBEDDINGS: TextEmbedderModel = { modelPath: `${BASE_URL}-paraphrase-multilingual-MiniLM-L12-v2/${NEXT_VERSION_TAG}/xnnpack/paraphrase_multilingual_minilm_l12_v2_xnnpack_8da4w.pte`, tokenizerPath: `${BASE_URL}-paraphrase-multilingual-MiniLM-L12-v2/${NEXT_VERSION_TAG}/tokenizer.json`, }; +const PARAPHRASE_MULTILINGUAL_MINILM_L12_V2_XNNPACK_FP32: TextEmbedderModel = { + modelPath: `${BASE_URL}-paraphrase-multilingual-MiniLM-L12-v2/${NEXT_VERSION_TAG}/xnnpack/paraphrase_multilingual_minilm_l12_v2_xnnpack_fp32.pte`, + tokenizerPath: `${BASE_URL}-paraphrase-multilingual-MiniLM-L12-v2/${NEXT_VERSION_TAG}/tokenizer.json`, +}; +const PARAPHRASE_MULTILINGUAL_MINILM_L12_V2_COREML_FP16: TextEmbedderModel = { + modelPath: `${BASE_URL}-paraphrase-multilingual-MiniLM-L12-v2/${NEXT_VERSION_TAG}/coreml/paraphrase_multilingual_minilm_l12_v2_coreml_fp16.pte`, + tokenizerPath: `${BASE_URL}-paraphrase-multilingual-MiniLM-L12-v2/${NEXT_VERSION_TAG}/tokenizer.json`, +}; +const PARAPHRASE_MULTILINGUAL_MINILM_L12_V2_VULKAN_FP16: TextEmbedderModel = { + modelPath: `${BASE_URL}-paraphrase-multilingual-MiniLM-L12-v2/${NEXT_VERSION_TAG}/vulkan/paraphrase_multilingual_minilm_l12_v2_vulkan_fp16.pte`, + tokenizerPath: `${BASE_URL}-paraphrase-multilingual-MiniLM-L12-v2/${NEXT_VERSION_TAG}/tokenizer.json`, +}; const DISTILUSE_BASE_MULTILINGUAL_CASED_V2_EMBEDDINGS: TextEmbedderModel = { modelPath: `${BASE_URL}-distiluse-base-multilingual-cased-v2/${NEXT_VERSION_TAG}/xnnpack/distiluse_base_multilingual_cased_v2_xnnpack_8da4w.pte`, tokenizerPath: `${BASE_URL}-distiluse-base-multilingual-cased-v2/${NEXT_VERSION_TAG}/tokenizer.json`, }; +const DISTILUSE_BASE_MULTILINGUAL_CASED_V2_XNNPACK_FP32: TextEmbedderModel = { + modelPath: `${BASE_URL}-distiluse-base-multilingual-cased-v2/${NEXT_VERSION_TAG}/xnnpack/distiluse_base_multilingual_cased_v2_xnnpack_fp32.pte`, + tokenizerPath: `${BASE_URL}-distiluse-base-multilingual-cased-v2/${NEXT_VERSION_TAG}/tokenizer.json`, +}; +const DISTILUSE_BASE_MULTILINGUAL_CASED_V2_COREML_FP16: TextEmbedderModel = { + modelPath: `${BASE_URL}-distiluse-base-multilingual-cased-v2/${NEXT_VERSION_TAG}/coreml/distiluse_base_multilingual_cased_v2_coreml_fp16.pte`, + tokenizerPath: `${BASE_URL}-distiluse-base-multilingual-cased-v2/${NEXT_VERSION_TAG}/tokenizer.json`, +}; const DISTILUSE_BASE_MULTILINGUAL_CASED_V2_MLX_INT8: TextEmbedderModel = { modelPath: `${BASE_URL}-distiluse-base-multilingual-cased-v2/${NEXT_VERSION_TAG}/mlx/distiluse_base_multilingual_cased_v2_mlx_int8.pte`, tokenizerPath: `${BASE_URL}-distiluse-base-multilingual-cased-v2/${NEXT_VERSION_TAG}/tokenizer.json`, }; +const DISTILUSE_BASE_MULTILINGUAL_CASED_V2_VULKAN_FP16: TextEmbedderModel = { + modelPath: `${BASE_URL}-distiluse-base-multilingual-cased-v2/${NEXT_VERSION_TAG}/vulkan/distiluse_base_multilingual_cased_v2_vulkan_fp16.pte`, + tokenizerPath: `${BASE_URL}-distiluse-base-multilingual-cased-v2/${NEXT_VERSION_TAG}/tokenizer.json`, +}; const CLIP_VIT_BASE_PATCH32_TEXT_EMBEDDINGS: TextEmbedderModel = { modelPath: `${BASE_URL}-clip-vit-base-patch32/${NEXT_VERSION_TAG}/xnnpack/clip_vit_base_patch32_text_xnnpack_fp32.pte`, tokenizerPath: `${BASE_URL}-clip-vit-base-patch32/${NEXT_VERSION_TAG}/tokenizer.json`, }; +const CLIP_VIT_BASE_PATCH32_TEXT_COREML_FP16: TextEmbedderModel = { + modelPath: `${BASE_URL}-clip-vit-base-patch32/${NEXT_VERSION_TAG}/coreml/clip_vit_base_patch32_text_coreml_fp16.pte`, + tokenizerPath: `${BASE_URL}-clip-vit-base-patch32/${NEXT_VERSION_TAG}/tokenizer.json`, +}; +const CLIP_VIT_BASE_PATCH32_TEXT_VULKAN_FP16: TextEmbedderModel = { + modelPath: `${BASE_URL}-clip-vit-base-patch32/${NEXT_VERSION_TAG}/vulkan/clip_vit_base_patch32_text_vulkan_fp16.pte`, + tokenizerPath: `${BASE_URL}-clip-vit-base-patch32/${NEXT_VERSION_TAG}/tokenizer.json`, +}; const LFM2_5_EMBEDDING_350M_EMBEDDINGS: TextEmbedderModel = { modelPath: `${BASE_URL}-lfm2.5-embedding-350m/${NEXT_VERSION_TAG}/xnnpack/lfm_2_5_embedding_350m_xnnpack_8da4w.pte`, tokenizerPath: `${BASE_URL}-lfm2.5-embedding-350m/${NEXT_VERSION_TAG}/tokenizer.json`, @@ -682,6 +940,10 @@ const CLIP_VIT_BASE_PATCH32_IMAGE_MLX_INT8: ImageEmbedderModel = { modelPath: `${BASE_URL}-clip-vit-base-patch32/${NEXT_VERSION_TAG}/mlx/clip_vit_base_patch32_image_mlx_int8.pte`, modelOpts: CLIP_IMAGE_EMBEDDINGS_OPTS, }; +const CLIP_VIT_BASE_PATCH32_IMAGE_VULKAN_FP16: ImageEmbedderModel = { + modelPath: `${BASE_URL}-clip-vit-base-patch32/${NEXT_VERSION_TAG}/vulkan/clip_vit_base_patch32_image_vulkan_fp16.pte`, + modelOpts: CLIP_IMAGE_EMBEDDINGS_OPTS, +}; // ============================================================================= // Voice Activity Detection @@ -706,6 +968,12 @@ const WHISPER_TINY_EN_XNNPACK_FP32: WhisperSttModel<'en'> = { supportedLanguages: ['en'], vadModel: FSMN_VAD_XNNPACK_FP32, }; +const WHISPER_TINY_EN_XNNPACK_INT8: WhisperSttModel<'en'> = { + modelPath: `${BASE_URL}-whisper-tiny.en/${NEXT_VERSION_TAG}/xnnpack/whisper_tiny_en_xnnpack_int8.pte`, + tokenizerPath: `${BASE_URL}-whisper-tiny.en/${NEXT_VERSION_TAG}/tokenizer.json`, + supportedLanguages: ['en'], + vadModel: FSMN_VAD_XNNPACK_FP32, +}; const WHISPER_TINY_EN_COREML_FP16: WhisperSttModel<'en'> = { modelPath: `${BASE_URL}-whisper-tiny.en/${NEXT_VERSION_TAG}/coreml/whisper_tiny_en_coreml_fp16.pte`, tokenizerPath: `${BASE_URL}-whisper-tiny.en/${NEXT_VERSION_TAG}/tokenizer.json`, @@ -724,6 +992,18 @@ const WHISPER_TINY_EN_MLX_INT8: WhisperSttModel<'en'> = { supportedLanguages: ['en'], vadModel: FSMN_VAD_XNNPACK_FP32, }; +const WHISPER_TINY_EN_VULKAN_FP16: WhisperSttModel<'en'> = { + modelPath: `${BASE_URL}-whisper-tiny.en/${NEXT_VERSION_TAG}/vulkan/whisper_tiny_en_vulkan_fp16.pte`, + tokenizerPath: `${BASE_URL}-whisper-tiny.en/${NEXT_VERSION_TAG}/tokenizer.json`, + supportedLanguages: ['en'], + vadModel: FSMN_VAD_XNNPACK_FP32, +}; +const WHISPER_TINY_EN_VULKAN_INT8: WhisperSttModel<'en'> = { + modelPath: `${BASE_URL}-whisper-tiny.en/${NEXT_VERSION_TAG}/vulkan/whisper_tiny_en_vulkan_int8.pte`, + tokenizerPath: `${BASE_URL}-whisper-tiny.en/${NEXT_VERSION_TAG}/tokenizer.json`, + supportedLanguages: ['en'], + vadModel: FSMN_VAD_XNNPACK_FP32, +}; const WHISPER_TINY_XNNPACK_FP32: WhisperSttModel = { modelPath: `${BASE_URL}-whisper-tiny/${NEXT_VERSION_TAG}/xnnpack/whisper_tiny_xnnpack_fp32.pte`, @@ -749,6 +1029,18 @@ const WHISPER_TINY_MLX_INT8: WhisperSttModel = { supportedLanguages: WHISPER_LANGUAGES, vadModel: FSMN_VAD_XNNPACK_FP32, }; +const WHISPER_TINY_VULKAN_FP16: WhisperSttModel = { + modelPath: `${BASE_URL}-whisper-tiny/${NEXT_VERSION_TAG}/vulkan/whisper_tiny_vulkan_fp16.pte`, + tokenizerPath: `${BASE_URL}-whisper-tiny/${NEXT_VERSION_TAG}/tokenizer.json`, + supportedLanguages: WHISPER_LANGUAGES, + vadModel: FSMN_VAD_XNNPACK_FP32, +}; +const WHISPER_TINY_VULKAN_INT8: WhisperSttModel = { + modelPath: `${BASE_URL}-whisper-tiny/${NEXT_VERSION_TAG}/vulkan/whisper_tiny_vulkan_int8.pte`, + tokenizerPath: `${BASE_URL}-whisper-tiny/${NEXT_VERSION_TAG}/tokenizer.json`, + supportedLanguages: WHISPER_LANGUAGES, + vadModel: FSMN_VAD_XNNPACK_FP32, +}; const WHISPER_BASE_EN_XNNPACK_FP32: WhisperSttModel<'en'> = { modelPath: `${BASE_URL}-whisper-base.en/${NEXT_VERSION_TAG}/xnnpack/whisper_base_en_xnnpack_fp32.pte`, @@ -756,6 +1048,12 @@ const WHISPER_BASE_EN_XNNPACK_FP32: WhisperSttModel<'en'> = { supportedLanguages: ['en'], vadModel: FSMN_VAD_XNNPACK_FP32, }; +const WHISPER_BASE_EN_XNNPACK_INT8: WhisperSttModel<'en'> = { + modelPath: `${BASE_URL}-whisper-base.en/${NEXT_VERSION_TAG}/xnnpack/whisper_base_en_xnnpack_int8.pte`, + tokenizerPath: `${BASE_URL}-whisper-base.en/${NEXT_VERSION_TAG}/tokenizer.json`, + supportedLanguages: ['en'], + vadModel: FSMN_VAD_XNNPACK_FP32, +}; const WHISPER_BASE_EN_COREML_FP16: WhisperSttModel<'en'> = { modelPath: `${BASE_URL}-whisper-base.en/${NEXT_VERSION_TAG}/coreml/whisper_base_en_coreml_fp16.pte`, tokenizerPath: `${BASE_URL}-whisper-base.en/${NEXT_VERSION_TAG}/tokenizer.json`, @@ -774,6 +1072,18 @@ const WHISPER_BASE_EN_MLX_INT8: WhisperSttModel<'en'> = { supportedLanguages: ['en'], vadModel: FSMN_VAD_XNNPACK_FP32, }; +const WHISPER_BASE_EN_VULKAN_FP16: WhisperSttModel<'en'> = { + modelPath: `${BASE_URL}-whisper-base.en/${NEXT_VERSION_TAG}/vulkan/whisper_base_en_vulkan_fp16.pte`, + tokenizerPath: `${BASE_URL}-whisper-base.en/${NEXT_VERSION_TAG}/tokenizer.json`, + supportedLanguages: ['en'], + vadModel: FSMN_VAD_XNNPACK_FP32, +}; +const WHISPER_BASE_EN_VULKAN_INT8: WhisperSttModel<'en'> = { + modelPath: `${BASE_URL}-whisper-base.en/${NEXT_VERSION_TAG}/vulkan/whisper_base_en_vulkan_int8.pte`, + tokenizerPath: `${BASE_URL}-whisper-base.en/${NEXT_VERSION_TAG}/tokenizer.json`, + supportedLanguages: ['en'], + vadModel: FSMN_VAD_XNNPACK_FP32, +}; const WHISPER_BASE_XNNPACK_FP32: WhisperSttModel = { modelPath: `${BASE_URL}-whisper-base/${NEXT_VERSION_TAG}/xnnpack/whisper_base_xnnpack_fp32.pte`, @@ -799,6 +1109,18 @@ const WHISPER_BASE_MLX_INT8: WhisperSttModel = { supportedLanguages: WHISPER_LANGUAGES, vadModel: FSMN_VAD_XNNPACK_FP32, }; +const WHISPER_BASE_VULKAN_FP16: WhisperSttModel = { + modelPath: `${BASE_URL}-whisper-base/${NEXT_VERSION_TAG}/vulkan/whisper_base_vulkan_fp16.pte`, + tokenizerPath: `${BASE_URL}-whisper-base/${NEXT_VERSION_TAG}/tokenizer.json`, + supportedLanguages: WHISPER_LANGUAGES, + vadModel: FSMN_VAD_XNNPACK_FP32, +}; +const WHISPER_BASE_VULKAN_INT8: WhisperSttModel = { + modelPath: `${BASE_URL}-whisper-base/${NEXT_VERSION_TAG}/vulkan/whisper_base_vulkan_int8.pte`, + tokenizerPath: `${BASE_URL}-whisper-base/${NEXT_VERSION_TAG}/tokenizer.json`, + supportedLanguages: WHISPER_LANGUAGES, + vadModel: FSMN_VAD_XNNPACK_FP32, +}; const WHISPER_SMALL_EN_XNNPACK_FP32: WhisperSttModel<'en'> = { modelPath: `${BASE_URL}-whisper-small.en/${NEXT_VERSION_TAG}/xnnpack/whisper_small_en_xnnpack_fp32.pte`, @@ -806,6 +1128,12 @@ const WHISPER_SMALL_EN_XNNPACK_FP32: WhisperSttModel<'en'> = { supportedLanguages: ['en'], vadModel: FSMN_VAD_XNNPACK_FP32, }; +const WHISPER_SMALL_EN_XNNPACK_INT8: WhisperSttModel<'en'> = { + modelPath: `${BASE_URL}-whisper-small.en/${NEXT_VERSION_TAG}/xnnpack/whisper_small_en_xnnpack_int8.pte`, + tokenizerPath: `${BASE_URL}-whisper-small.en/${NEXT_VERSION_TAG}/tokenizer.json`, + supportedLanguages: ['en'], + vadModel: FSMN_VAD_XNNPACK_FP32, +}; const WHISPER_SMALL_EN_COREML_FP16: WhisperSttModel<'en'> = { modelPath: `${BASE_URL}-whisper-small.en/${NEXT_VERSION_TAG}/coreml/whisper_small_en_coreml_fp16.pte`, tokenizerPath: `${BASE_URL}-whisper-small.en/${NEXT_VERSION_TAG}/tokenizer.json`, @@ -818,6 +1146,18 @@ const WHISPER_SMALL_EN_MLX_INT8: WhisperSttModel<'en'> = { supportedLanguages: ['en'], vadModel: FSMN_VAD_XNNPACK_FP32, }; +const WHISPER_SMALL_EN_VULKAN_FP16: WhisperSttModel<'en'> = { + modelPath: `${BASE_URL}-whisper-small.en/${NEXT_VERSION_TAG}/vulkan/whisper_small_en_vulkan_fp16.pte`, + tokenizerPath: `${BASE_URL}-whisper-small.en/${NEXT_VERSION_TAG}/tokenizer.json`, + supportedLanguages: ['en'], + vadModel: FSMN_VAD_XNNPACK_FP32, +}; +const WHISPER_SMALL_EN_VULKAN_INT8: WhisperSttModel<'en'> = { + modelPath: `${BASE_URL}-whisper-small.en/${NEXT_VERSION_TAG}/vulkan/whisper_small_en_vulkan_int8.pte`, + tokenizerPath: `${BASE_URL}-whisper-small.en/${NEXT_VERSION_TAG}/tokenizer.json`, + supportedLanguages: ['en'], + vadModel: FSMN_VAD_XNNPACK_FP32, +}; const WHISPER_SMALL_XNNPACK_FP32: WhisperSttModel = { modelPath: `${BASE_URL}-whisper-small/${NEXT_VERSION_TAG}/xnnpack/whisper_small_xnnpack_fp32.pte`, @@ -837,6 +1177,18 @@ const WHISPER_SMALL_MLX_INT8: WhisperSttModel = { supportedLanguages: WHISPER_LANGUAGES, vadModel: FSMN_VAD_XNNPACK_FP32, }; +const WHISPER_SMALL_VULKAN_FP16: WhisperSttModel = { + modelPath: `${BASE_URL}-whisper-small/${NEXT_VERSION_TAG}/vulkan/whisper_small_vulkan_fp16.pte`, + tokenizerPath: `${BASE_URL}-whisper-small/${NEXT_VERSION_TAG}/tokenizer.json`, + supportedLanguages: WHISPER_LANGUAGES, + vadModel: FSMN_VAD_XNNPACK_FP32, +}; +const WHISPER_SMALL_VULKAN_INT8: WhisperSttModel = { + modelPath: `${BASE_URL}-whisper-small/${NEXT_VERSION_TAG}/vulkan/whisper_small_vulkan_int8.pte`, + tokenizerPath: `${BASE_URL}-whisper-small/${NEXT_VERSION_TAG}/tokenizer.json`, + supportedLanguages: WHISPER_LANGUAGES, + vadModel: FSMN_VAD_XNNPACK_FP32, +}; // ============================================================================= // Text to Image @@ -886,6 +1238,18 @@ const SUPERTONIC_3_MLX_FP32: SupertonicTtsModel = { voiceStyles: SUPERTONIC_DEFAULT_VOICE_STYLES, }; +const SUPERTONIC_3_VULKAN_FP16: SupertonicTtsModel = { + name: 'supertonic', + modelPaths: { + durationPredictor: `${BASE_URL}-supertonic/${NEXT_VERSION_TAG}/vulkan/duration_predictor_vulkan_fp16.pte`, + vectorEstimator: `${BASE_URL}-supertonic/${NEXT_VERSION_TAG}/vulkan/vector_estimator_vulkan_fp16.pte`, + textEncoder: `${BASE_URL}-supertonic/${NEXT_VERSION_TAG}/vulkan/text_encoder_vulkan_fp16.pte`, + vocoder: `${BASE_URL}-supertonic/${NEXT_VERSION_TAG}/vulkan/vocoder_vulkan_fp16.pte`, + }, + unicodeIndexerPath: `${BASE_URL}-supertonic/${NEXT_VERSION_TAG}/unicode_indexer.json`, + voiceStyles: SUPERTONIC_DEFAULT_VOICE_STYLES, +}; + const KOKORO_ROOT = `${BASE_URL}-kokoro/${NEXT_VERSION_TAG}`; const KOKORO_PHONEMIZER_ROOT = `${KOKORO_ROOT}/phonemizer`; @@ -964,7 +1328,6 @@ const KOKORO_HI_XNNPACK_FP32: KokoroTtsModel<'hf_alpha' | 'hm_omega' | 'hm_psi'> phonemizer: kokoroNeuralPhonemizer('hi'), voices: kokoroVoices(['hf_alpha', 'hm_omega', 'hm_psi']), }; - const KOKORO_PL_XNNPACK_FP32: KokoroTtsModel<'pm_mateusz'> = { name: 'kokoro', modelPaths: kokoroModelPaths('xnnpack', 'pl', 'polish'), @@ -1100,6 +1463,11 @@ const PPOCRV6_SMALL_XNNPACK_INT8: PaddleOcrModel = { charsetPath: PPOCRV6_CHARSET, modelOpts: PPOCRV6_OPTS, }; +const PPOCRV6_SMALL_XNNPACK_FP32: PaddleOcrModel = { + modelPath: `${BASE_URL}-pp-ocrv6/${NEXT_VERSION_TAG}/xnnpack/pp_ocrv6_xnnpack_fp32.pte`, + charsetPath: PPOCRV6_CHARSET, + modelOpts: PPOCRV6_OPTS, +}; const PPOCRV6_SMALL_COREML_INT8: PaddleOcrModel = { modelPath: `${BASE_URL}-pp-ocrv6/${NEXT_VERSION_TAG}/coreml/pp_ocrv6_coreml_int8.pte`, charsetPath: PPOCRV6_CHARSET, @@ -1193,6 +1561,20 @@ const LFM2_5_VL_1_6B_VULKAN_8DA4W: LLMModel = { modalities: ['image'], preprocessorConfig: LFM2_5_VL_PREPROCESSOR_CONFIG, }; +const LFM2_5_VL_1_6B_MLX_INT4: LLMModel = { + modelPath: `${LFM2_5_BASE_URL}/vl_1_6b/mlx/lfm_2_5_vl_1_6b_mlx_int4.pte`, + tokenizerPath: `${LFM2_5_BASE_URL}/vl_1_6b/tokenizer.json`, + tokenizerConfigPath: `${LFM2_5_BASE_URL}/vl_1_6b/tokenizer_config.json`, + modalities: ['image'], + preprocessorConfig: LFM2_5_VL_PREPROCESSOR_CONFIG, +}; +const LFM2_5_VL_1_6B_MLX_INT8: LLMModel = { + modelPath: `${LFM2_5_BASE_URL}/vl_1_6b/mlx/lfm_2_5_vl_1_6b_mlx_int8.pte`, + tokenizerPath: `${LFM2_5_BASE_URL}/vl_1_6b/tokenizer.json`, + tokenizerConfigPath: `${LFM2_5_BASE_URL}/vl_1_6b/tokenizer_config.json`, + modalities: ['image'], + preprocessorConfig: LFM2_5_VL_PREPROCESSOR_CONFIG, +}; const BIELIK_V3_1_5B_BASE_URL = `${BASE_URL}-bielik-v3.0/${NEXT_VERSION_TAG}`; @@ -1334,6 +1716,11 @@ const GEMMA4_E2B_XNNPACK_8DA4W: LLMModel = { tokenizerPath: `${GEMMA4_BASE_URL}/e2b/tokenizer.json`, tokenizerConfigPath: `${GEMMA4_BASE_URL}/e2b/tokenizer_config.json`, }; +const GEMMA4_E2B_VULKAN_8DA4W: LLMModel = { + modelPath: `${GEMMA4_BASE_URL}/e2b/vulkan/gemma_4_e2b_vulkan_8da4w.pte`, + tokenizerPath: `${GEMMA4_BASE_URL}/e2b/tokenizer.json`, + tokenizerConfigPath: `${GEMMA4_BASE_URL}/e2b/tokenizer_config.json`, +}; const GEMMA4_E2B_MLX_INT4: LLMModel = { modelPath: `${GEMMA4_BASE_URL}/e2b/mlx/gemma4_e2b_mlx_int4.pte`, tokenizerPath: `${GEMMA4_BASE_URL}/e2b/tokenizer.json`, @@ -1379,6 +1766,13 @@ const QWEN3_4B_XNNPACK_BF16: LLMModel = { * This provides Hugging Face repository URLs and baseline configurations for * tasks, allowing quick model loading and execution without manual option * setup. + * + * Models published for more than one backend expose their exports as named + * variants (`XNNPACK_INT8`, `COREML_FP16`, ...) plus a `DEFAULT` alias. The + * alias is chosen for the device the app runs on: Core ML then MLX on iOS + * hardware, Vulkan on Android, XNNPACK as the fallback everywhere and the only + * option on the iOS simulator — always narrowed to the backends the app + * actually linked in. Reach for a named variant to override that. * @category Models */ export const models = { @@ -1393,12 +1787,11 @@ export const models = { * architecture providing high accuracy for general-purpose image * classification. */ - EFFICIENTNET_V2_S: { - DEFAULT: EFFICIENTNET_V2_S_XNNPACK_INT8, + EFFICIENTNET_V2_S: variants({ XNNPACK_INT8: EFFICIENTNET_V2_S_XNNPACK_INT8, XNNPACK_FP32: EFFICIENTNET_V2_S_XNNPACK_FP32, COREML_FP16: EFFICIENTNET_V2_S_COREML_FP16, - }, + }), }, /** @@ -1410,42 +1803,38 @@ export const models = { * Fast neural style transfer model generating a vibrant, artistic "Candy" * style effect. */ - CANDY: { - DEFAULT: STYLE_TRANSFER_CANDY_XNNPACK_INT8, - XNNPACK_FP32: STYLE_TRANSFER_CANDY_XNNPACK_FP32, + CANDY: variants({ XNNPACK_INT8: STYLE_TRANSFER_CANDY_XNNPACK_INT8, + XNNPACK_FP32: STYLE_TRANSFER_CANDY_XNNPACK_FP32, COREML_FP16: STYLE_TRANSFER_CANDY_COREML_FP16, - }, + }), /** * Fast neural style transfer model applying a classic tile mosaic artistic * pattern. */ - MOSAIC: { - DEFAULT: STYLE_TRANSFER_MOSAIC_XNNPACK_INT8, - XNNPACK_FP32: STYLE_TRANSFER_MOSAIC_XNNPACK_FP32, + MOSAIC: variants({ XNNPACK_INT8: STYLE_TRANSFER_MOSAIC_XNNPACK_INT8, + XNNPACK_FP32: STYLE_TRANSFER_MOSAIC_XNNPACK_FP32, COREML_FP16: STYLE_TRANSFER_MOSAIC_COREML_FP16, - }, + }), /** * Fast neural style transfer model applying a painterly "Rain Princess" oil * painting aesthetic. */ - RAIN_PRINCESS: { - DEFAULT: STYLE_TRANSFER_RAIN_PRINCESS_XNNPACK_INT8, - XNNPACK_FP32: STYLE_TRANSFER_RAIN_PRINCESS_XNNPACK_FP32, + RAIN_PRINCESS: variants({ XNNPACK_INT8: STYLE_TRANSFER_RAIN_PRINCESS_XNNPACK_INT8, + XNNPACK_FP32: STYLE_TRANSFER_RAIN_PRINCESS_XNNPACK_FP32, COREML_FP16: STYLE_TRANSFER_RAIN_PRINCESS_COREML_FP16, - }, + }), /** * Fast neural style transfer model applying Francis Picabia's "Udnie" * abstract art style. */ - UDNIE: { - DEFAULT: STYLE_TRANSFER_UDNIE_XNNPACK_INT8, - XNNPACK_FP32: STYLE_TRANSFER_UDNIE_XNNPACK_FP32, + UDNIE: variants({ XNNPACK_INT8: STYLE_TRANSFER_UDNIE_XNNPACK_INT8, + XNNPACK_FP32: STYLE_TRANSFER_UDNIE_XNNPACK_FP32, COREML_FP16: STYLE_TRANSFER_UDNIE_COREML_FP16, - }, + }), }, /** @@ -1458,84 +1847,76 @@ export const models = { * background separation. Categorizes pixels into `background` and `person`. * Ideal for background blur and replacement effects. */ - SELFIE_SEGMENTATION: { - DEFAULT: SELFIE_SEGMENTATION_XNNPACK_FP32, + SELFIE_SEGMENTATION: variants({ XNNPACK_FP32: SELFIE_SEGMENTATION_XNNPACK_FP32, COREML_FP16: SELFIE_SEGMENTATION_COREML_FP16, - }, + }), /** * MediaPipe Selfie Segmentation, landscape orientation. A separate * 256x144 checkpoint rather than a resize of the portrait model. */ - SELFIE_SEGMENTATION_LANDSCAPE: { - DEFAULT: SELFIE_SEGMENTATION_LANDSCAPE_XNNPACK_FP32, + SELFIE_SEGMENTATION_LANDSCAPE: variants({ XNNPACK_FP32: SELFIE_SEGMENTATION_LANDSCAPE_XNNPACK_FP32, COREML_FP16: SELFIE_SEGMENTATION_LANDSCAPE_COREML_FP16, - }, + }), /** * Lite R-ASPP semantic segmentation model with MobileNetV3-Large backbone * (21 classes, see {@link PASCAL_VOC_LABELS}). Optimized for low-latency, * real-time pixel-level segmentation on mobile devices. */ - LRASPP_MOBILENET_V3_LARGE: { - DEFAULT: LRASPP_MOBILENET_V3_LARGE_XNNPACK_INT8, - XNNPACK_FP32: LRASPP_MOBILENET_V3_LARGE_XNNPACK_FP32, + LRASPP_MOBILENET_V3_LARGE: variants({ XNNPACK_INT8: LRASPP_MOBILENET_V3_LARGE_XNNPACK_INT8, + XNNPACK_FP32: LRASPP_MOBILENET_V3_LARGE_XNNPACK_FP32, COREML_FP16: LRASPP_MOBILENET_V3_LARGE_COREML_FP16, - }, + }), /** * DeepLabV3 semantic segmentation model with ResNet-50 backbone (21 * classes, see {@link PASCAL_VOC_LABELS}). High-accuracy segmentation * utilizing atrous spatial pyramid pooling. */ - DEEPLAB_V3_RESNET50: { - DEFAULT: DEEPLAB_V3_RESNET50_XNNPACK_INT8, - XNNPACK_FP32: DEEPLAB_V3_RESNET50_XNNPACK_FP32, + DEEPLAB_V3_RESNET50: variants({ XNNPACK_INT8: DEEPLAB_V3_RESNET50_XNNPACK_INT8, + XNNPACK_FP32: DEEPLAB_V3_RESNET50_XNNPACK_FP32, COREML_FP16: DEEPLAB_V3_RESNET50_COREML_FP16, - }, + }), /** * DeepLabV3 semantic segmentation model with ResNet-101 backbone (21 * classes, see {@link PASCAL_VOC_LABELS}). High-capacity backbone for * maximum segmentation detail and boundary accuracy. */ - DEEPLAB_V3_RESNET101: { - DEFAULT: DEEPLAB_V3_RESNET101_XNNPACK_INT8, - XNNPACK_FP32: DEEPLAB_V3_RESNET101_XNNPACK_FP32, + DEEPLAB_V3_RESNET101: variants({ XNNPACK_INT8: DEEPLAB_V3_RESNET101_XNNPACK_INT8, + XNNPACK_FP32: DEEPLAB_V3_RESNET101_XNNPACK_FP32, COREML_FP16: DEEPLAB_V3_RESNET101_COREML_FP16, - }, + }), /** * DeepLabV3 semantic segmentation model with MobileNetV3-Large backbone (21 * classes, see {@link PASCAL_VOC_LABELS}). Combines DeepLabV3 feature * extraction quality with a lightweight mobile backbone. */ - DEEPLAB_V3_MOBILENET_V3_LARGE: { - DEFAULT: DEEPLAB_V3_MOBILENET_V3_LARGE_XNNPACK_INT8, - XNNPACK_FP32: DEEPLAB_V3_MOBILENET_V3_LARGE_XNNPACK_FP32, + DEEPLAB_V3_MOBILENET_V3_LARGE: variants({ XNNPACK_INT8: DEEPLAB_V3_MOBILENET_V3_LARGE_XNNPACK_INT8, + XNNPACK_FP32: DEEPLAB_V3_MOBILENET_V3_LARGE_XNNPACK_FP32, COREML_FP16: DEEPLAB_V3_MOBILENET_V3_LARGE_COREML_FP16, - }, + }), /** * Fully Convolutional Network (FCN) semantic segmentation model with * ResNet-50 backbone (21 classes, see {@link PASCAL_VOC_LABELS}). */ - FCN_RESNET50: { - DEFAULT: FCN_RESNET50_XNNPACK_INT8, - XNNPACK_FP32: FCN_RESNET50_XNNPACK_FP32, + FCN_RESNET50: variants({ XNNPACK_INT8: FCN_RESNET50_XNNPACK_INT8, + XNNPACK_FP32: FCN_RESNET50_XNNPACK_FP32, COREML_FP16: FCN_RESNET50_COREML_FP16, - }, + }), /** * Fully Convolutional Network (FCN) semantic segmentation model with * ResNet-101 backbone (21 classes, see {@link PASCAL_VOC_LABELS}). */ - FCN_RESNET101: { - DEFAULT: FCN_RESNET101_XNNPACK_INT8, - XNNPACK_FP32: FCN_RESNET101_XNNPACK_FP32, + FCN_RESNET101: variants({ XNNPACK_INT8: FCN_RESNET101_XNNPACK_INT8, + XNNPACK_FP32: FCN_RESNET101_XNNPACK_FP32, COREML_FP16: FCN_RESNET101_COREML_FP16, - }, + }), }, /** @@ -1547,139 +1928,116 @@ export const models = { * (see {@link COCO_CLASSES}) at 320x320 resolution. Fast, lightweight * detector suited for real-time mobile applications. */ - SSDLITE320_MOBILENET_V3_LARGE: { - DEFAULT: SSDLITE320_MOBILENET_V3_LARGE_XNNPACK_FP32, + SSDLITE320_MOBILENET_V3_LARGE: variants({ XNNPACK_FP32: SSDLITE320_MOBILENET_V3_LARGE_XNNPACK_FP32, COREML_FP16: SSDLITE320_MOBILENET_V3_LARGE_COREML_FP16, - }, + }), /** * RF-DETR (Roboflow Detection Transformer) Nano variant trained on COCO * (see {@link COCO_CLASSES}). Modern end-to-end DINOv2-based transformer * object detector. */ - RFDETR_NANO: { - DEFAULT: RFDETR_NANO_DETECTOR_XNNPACK_FP32, + RFDETR_NANO: variants({ XNNPACK_FP32: RFDETR_NANO_DETECTOR_XNNPACK_FP32, COREML_FP16: RFDETR_NANO_DETECTOR_COREML_FP16, - }, + }), /** * Ultralytics YOLO26 real-time object detection models trained on COCO (80 * classes, see {@link COCO_CLASSES_YOLO}). Available across multiple scale * sizes (NANO, SMALL, MEDIUM, LARGE, XLARGE) and resolutions (384x384, * 512x512, 640x640). */ - YOLO26: { - DEFAULT: YOLO26_NANO_384_XNNPACK_FP32, + YOLO26: family({ /** * Nano scale YOLO26 object detection model. High speed, ultra low * latency. */ - NANO: { - DEFAULT: YOLO26_NANO_384_XNNPACK_FP32, - SIZE_384: { - DEFAULT: YOLO26_NANO_384_XNNPACK_FP32, + NANO: family({ + SIZE_384: variants({ XNNPACK_FP32: YOLO26_NANO_384_XNNPACK_FP32, COREML_FP16: YOLO26_NANO_384_COREML_FP16, - }, - SIZE_512: { - DEFAULT: YOLO26_NANO_512_XNNPACK_FP32, + }), + SIZE_512: variants({ XNNPACK_FP32: YOLO26_NANO_512_XNNPACK_FP32, COREML_FP16: YOLO26_NANO_512_COREML_FP16, - }, - SIZE_640: { - DEFAULT: YOLO26_NANO_640_XNNPACK_FP32, + }), + SIZE_640: variants({ XNNPACK_FP32: YOLO26_NANO_640_XNNPACK_FP32, COREML_FP16: YOLO26_NANO_640_COREML_FP16, - }, - }, + }), + }), /** * Small scale YOLO26 object detection model. Balanced latency and * accuracy. */ - SMALL: { - DEFAULT: YOLO26_SMALL_384_XNNPACK_FP32, - SIZE_384: { - DEFAULT: YOLO26_SMALL_384_XNNPACK_FP32, + SMALL: family({ + SIZE_384: variants({ XNNPACK_FP32: YOLO26_SMALL_384_XNNPACK_FP32, COREML_FP16: YOLO26_SMALL_384_COREML_FP16, - }, - SIZE_512: { - DEFAULT: YOLO26_SMALL_512_XNNPACK_FP32, + }), + SIZE_512: variants({ XNNPACK_FP32: YOLO26_SMALL_512_XNNPACK_FP32, COREML_FP16: YOLO26_SMALL_512_COREML_FP16, - }, - SIZE_640: { - DEFAULT: YOLO26_SMALL_640_XNNPACK_FP32, + }), + SIZE_640: variants({ XNNPACK_FP32: YOLO26_SMALL_640_XNNPACK_FP32, COREML_FP16: YOLO26_SMALL_640_COREML_FP16, - }, - }, + }), + }), /** * Medium scale YOLO26 object detection model. Higher precision for * complex scenes. */ - MEDIUM: { - DEFAULT: YOLO26_MEDIUM_384_XNNPACK_FP32, - SIZE_384: { - DEFAULT: YOLO26_MEDIUM_384_XNNPACK_FP32, + MEDIUM: family({ + SIZE_384: variants({ XNNPACK_FP32: YOLO26_MEDIUM_384_XNNPACK_FP32, COREML_FP16: YOLO26_MEDIUM_384_COREML_FP16, - }, - SIZE_512: { - DEFAULT: YOLO26_MEDIUM_512_XNNPACK_FP32, + }), + SIZE_512: variants({ XNNPACK_FP32: YOLO26_MEDIUM_512_XNNPACK_FP32, COREML_FP16: YOLO26_MEDIUM_512_COREML_FP16, - }, - SIZE_640: { - DEFAULT: YOLO26_MEDIUM_640_XNNPACK_FP32, + }), + SIZE_640: variants({ XNNPACK_FP32: YOLO26_MEDIUM_640_XNNPACK_FP32, COREML_FP16: YOLO26_MEDIUM_640_COREML_FP16, - }, - }, + }), + }), /** * Large scale YOLO26 object detection model. High accuracy model variant. */ - LARGE: { - DEFAULT: YOLO26_LARGE_384_XNNPACK_FP32, - SIZE_384: { - DEFAULT: YOLO26_LARGE_384_XNNPACK_FP32, + LARGE: family({ + SIZE_384: variants({ XNNPACK_FP32: YOLO26_LARGE_384_XNNPACK_FP32, COREML_FP16: YOLO26_LARGE_384_COREML_FP16, - }, - SIZE_512: { - DEFAULT: YOLO26_LARGE_512_XNNPACK_FP32, + }), + SIZE_512: variants({ XNNPACK_FP32: YOLO26_LARGE_512_XNNPACK_FP32, COREML_FP16: YOLO26_LARGE_512_COREML_FP16, - }, - SIZE_640: { - DEFAULT: YOLO26_LARGE_640_XNNPACK_FP32, + }), + SIZE_640: variants({ XNNPACK_FP32: YOLO26_LARGE_640_XNNPACK_FP32, COREML_FP16: YOLO26_LARGE_640_COREML_FP16, - }, - }, + }), + }), /** * Extra Large scale YOLO26 object detection model. Maximum detection * performance. */ - XLARGE: { - DEFAULT: YOLO26_XLARGE_384_XNNPACK_FP32, - SIZE_384: { - DEFAULT: YOLO26_XLARGE_384_XNNPACK_FP32, + XLARGE: family({ + SIZE_384: variants({ XNNPACK_FP32: YOLO26_XLARGE_384_XNNPACK_FP32, COREML_FP16: YOLO26_XLARGE_384_COREML_FP16, - }, - SIZE_512: { - DEFAULT: YOLO26_XLARGE_512_XNNPACK_FP32, + }), + SIZE_512: variants({ XNNPACK_FP32: YOLO26_XLARGE_512_XNNPACK_FP32, COREML_FP16: YOLO26_XLARGE_512_COREML_FP16, - }, - SIZE_640: { - DEFAULT: YOLO26_XLARGE_640_XNNPACK_FP32, + }), + SIZE_640: variants({ XNNPACK_FP32: YOLO26_XLARGE_640_XNNPACK_FP32, COREML_FP16: YOLO26_XLARGE_640_COREML_FP16, - }, - }, - }, + }), + }), + }), }, /** @@ -1692,40 +2050,46 @@ export const models = { * landmark locator (eyes, nose, mouth, ears, see * {@link BLAZEFACE_LANDMARKS}). */ - BLAZEFACE: { - DEFAULT: BLAZEFACE_XNNPACK_FP32, + BLAZEFACE: variants({ XNNPACK_FP32: BLAZEFACE_XNNPACK_FP32, - }, + }), /** * YOLO26 human pose estimation model predicting 17 COCO body keypoints (see * {@link COCO_LANDMARKS}). Available across 384x384, 512x512, and 640x640 * resolutions. */ - YOLO26_POSE: { - DEFAULT: YOLO26_POSE_384_XNNPACK_FP32, - SIZE_384: { - DEFAULT: YOLO26_POSE_384_XNNPACK_FP32, + YOLO26_POSE: family({ + SIZE_384: variants({ XNNPACK_FP32: YOLO26_POSE_384_XNNPACK_FP32, - }, - SIZE_512: { - DEFAULT: YOLO26_POSE_512_XNNPACK_FP32, + COREML_FP16: YOLO26_POSE_384_COREML_FP16, + }), + SIZE_512: variants({ XNNPACK_FP32: YOLO26_POSE_512_XNNPACK_FP32, - }, - SIZE_640: { - DEFAULT: YOLO26_POSE_640_XNNPACK_FP32, + COREML_FP16: YOLO26_POSE_512_COREML_FP16, + }), + SIZE_640: variants({ XNNPACK_FP32: YOLO26_POSE_640_XNNPACK_FP32, - }, - }, + COREML_FP16: YOLO26_POSE_640_COREML_FP16, + }), + }), /** * RF-DETR (Roboflow Detection Transformer) pose keypoint detector * predicting 17 COCO body keypoints (see {@link COCO_LANDMARKS}). */ - RFDETR_KEYPOINT: { - DEFAULT: RFDETR_KEYPOINT_XNNPACK_FP32, - XNNPACK_FP32: RFDETR_KEYPOINT_XNNPACK_FP32, - COREML_FP32: RFDETR_KEYPOINT_COREML_FP32, - MLX_FP32: RFDETR_KEYPOINT_MLX_FP32, - }, + RFDETR_KEYPOINT: variants( + { + XNNPACK_FP32: RFDETR_KEYPOINT_XNNPACK_FP32, + COREML_FP16: RFDETR_KEYPOINT_COREML_FP16, + MLX_FP32: RFDETR_KEYPOINT_MLX_FP32, + }, + // Core ML over MLX: 144.0 ms against 272.3 on an iPhone 16, at 263 MB + // against 1304 MB. fp16 matches the fp32 build it replaced (landmarks to + // 1.04 px over 13 photos) but only under the GPU-only compute unit and an + // iOS17 deployment target — every other combination degrades it, and a + // macOS check passes builds the device gets wrong. See export-scripts + // MR !18 before re-exporting. + { ios: 'COREML_FP16' } + ), }, /** @@ -1742,134 +2106,125 @@ export const models = { /** * FastSAM Small - lightweight instance segmenter for mobile. */ - S: { - DEFAULT: FASTSAM_S_XNNPACK_FP32, + S: variants({ XNNPACK_FP32: FASTSAM_S_XNNPACK_FP32, COREML_FP16: FASTSAM_S_COREML_FP16, - }, + }), /** * FastSAM Extra Large - high-accuracy instance segmenter. */ - X: { - DEFAULT: FASTSAM_X_XNNPACK_FP32, + X: variants({ XNNPACK_FP32: FASTSAM_X_XNNPACK_FP32, COREML_FP16: FASTSAM_X_COREML_FP16, - }, + }), }, /** * RF-DETR (Roboflow Detection Transformer) Nano instance segmentation model * predicting COCO class masks and bounding boxes (see * {@link COCO_CLASSES}). */ - RFDETR_NANO: { - DEFAULT: RFDETR_NANO_SEG_COREML_FP16, - COREML_FP16: RFDETR_NANO_SEG_COREML_FP16, + RFDETR_NANO: variants({ XNNPACK_FP32: RFDETR_NANO_SEG_XNNPACK_FP32, - }, + COREML_FP16: RFDETR_NANO_SEG_COREML_FP16, + }), /** * YOLO26 instance segmentation models predicting COCO class instance masks * and bounding boxes (see {@link COCO_CLASSES_YOLO}). Available across * multiple sizes (NANO, SMALL, MEDIUM, LARGE, XLARGE) and resolutions * (384x384, 512x512, 640x640). */ - YOLO26: { - DEFAULT: YOLO26_NANO_SEG_384_XNNPACK_FP32, + YOLO26: family({ /** * Nano scale YOLO26 instance segmentation model. High speed, ultra low * latency mask generation. */ - NANO: { - DEFAULT: YOLO26_NANO_SEG_384_XNNPACK_FP32, - SIZE_384: { - DEFAULT: YOLO26_NANO_SEG_384_XNNPACK_FP32, + NANO: family({ + SIZE_384: variants({ XNNPACK_FP32: YOLO26_NANO_SEG_384_XNNPACK_FP32, - }, - SIZE_512: { - DEFAULT: YOLO26_NANO_SEG_512_XNNPACK_FP32, + COREML_FP16: YOLO26_NANO_SEG_384_COREML_FP16, + }), + SIZE_512: variants({ XNNPACK_FP32: YOLO26_NANO_SEG_512_XNNPACK_FP32, - }, - SIZE_640: { - DEFAULT: YOLO26_NANO_SEG_640_XNNPACK_FP32, + COREML_FP16: YOLO26_NANO_SEG_512_COREML_FP16, + }), + SIZE_640: variants({ XNNPACK_FP32: YOLO26_NANO_SEG_640_XNNPACK_FP32, - }, - }, + COREML_FP16: YOLO26_NANO_SEG_640_COREML_FP16, + }), + }), /** * Small scale YOLO26 instance segmentation model. Balanced latency and * mask accuracy. */ - SMALL: { - DEFAULT: YOLO26_SMALL_SEG_384_XNNPACK_FP32, - SIZE_384: { - DEFAULT: YOLO26_SMALL_SEG_384_XNNPACK_FP32, + SMALL: family({ + SIZE_384: variants({ XNNPACK_FP32: YOLO26_SMALL_SEG_384_XNNPACK_FP32, - }, - SIZE_512: { - DEFAULT: YOLO26_SMALL_SEG_512_XNNPACK_FP32, + COREML_FP16: YOLO26_SMALL_SEG_384_COREML_FP16, + }), + SIZE_512: variants({ XNNPACK_FP32: YOLO26_SMALL_SEG_512_XNNPACK_FP32, - }, - SIZE_640: { - DEFAULT: YOLO26_SMALL_SEG_640_XNNPACK_FP32, + COREML_FP16: YOLO26_SMALL_SEG_512_COREML_FP16, + }), + SIZE_640: variants({ XNNPACK_FP32: YOLO26_SMALL_SEG_640_XNNPACK_FP32, - }, - }, + COREML_FP16: YOLO26_SMALL_SEG_640_COREML_FP16, + }), + }), /** * Medium scale YOLO26 instance segmentation model. Higher mask boundary * precision for complex multi-object scenes. */ - MEDIUM: { - DEFAULT: YOLO26_MEDIUM_SEG_384_XNNPACK_FP32, - SIZE_384: { - DEFAULT: YOLO26_MEDIUM_SEG_384_XNNPACK_FP32, + MEDIUM: family({ + SIZE_384: variants({ XNNPACK_FP32: YOLO26_MEDIUM_SEG_384_XNNPACK_FP32, - }, - SIZE_512: { - DEFAULT: YOLO26_MEDIUM_SEG_512_XNNPACK_FP32, + COREML_FP16: YOLO26_MEDIUM_SEG_384_COREML_FP16, + }), + SIZE_512: variants({ XNNPACK_FP32: YOLO26_MEDIUM_SEG_512_XNNPACK_FP32, - }, - SIZE_640: { - DEFAULT: YOLO26_MEDIUM_SEG_640_XNNPACK_FP32, + COREML_FP16: YOLO26_MEDIUM_SEG_512_COREML_FP16, + }), + SIZE_640: variants({ XNNPACK_FP32: YOLO26_MEDIUM_SEG_640_XNNPACK_FP32, - }, - }, + COREML_FP16: YOLO26_MEDIUM_SEG_640_COREML_FP16, + }), + }), /** * Large scale YOLO26 instance segmentation model. High accuracy instance * segmentation variant for demanding visual pipelines. */ - LARGE: { - DEFAULT: YOLO26_LARGE_SEG_384_XNNPACK_FP32, - SIZE_384: { - DEFAULT: YOLO26_LARGE_SEG_384_XNNPACK_FP32, + LARGE: family({ + SIZE_384: variants({ XNNPACK_FP32: YOLO26_LARGE_SEG_384_XNNPACK_FP32, - }, - SIZE_512: { - DEFAULT: YOLO26_LARGE_SEG_512_XNNPACK_FP32, + COREML_FP16: YOLO26_LARGE_SEG_384_COREML_FP16, + }), + SIZE_512: variants({ XNNPACK_FP32: YOLO26_LARGE_SEG_512_XNNPACK_FP32, - }, - SIZE_640: { - DEFAULT: YOLO26_LARGE_SEG_640_XNNPACK_FP32, + COREML_FP16: YOLO26_LARGE_SEG_512_COREML_FP16, + }), + SIZE_640: variants({ XNNPACK_FP32: YOLO26_LARGE_SEG_640_XNNPACK_FP32, - }, - }, + COREML_FP16: YOLO26_LARGE_SEG_640_COREML_FP16, + }), + }), /** * Extra Large scale YOLO26 instance segmentation model. Maximum instance * segmentation and mask delineation performance. */ - XLARGE: { - DEFAULT: YOLO26_XLARGE_SEG_384_XNNPACK_FP32, - SIZE_384: { - DEFAULT: YOLO26_XLARGE_SEG_384_XNNPACK_FP32, + XLARGE: family({ + SIZE_384: variants({ XNNPACK_FP32: YOLO26_XLARGE_SEG_384_XNNPACK_FP32, - }, - SIZE_512: { - DEFAULT: YOLO26_XLARGE_SEG_512_XNNPACK_FP32, + COREML_FP16: YOLO26_XLARGE_SEG_384_COREML_FP16, + }), + SIZE_512: variants({ XNNPACK_FP32: YOLO26_XLARGE_SEG_512_XNNPACK_FP32, - }, - SIZE_640: { - DEFAULT: YOLO26_XLARGE_SEG_640_XNNPACK_FP32, + COREML_FP16: YOLO26_XLARGE_SEG_512_COREML_FP16, + }), + SIZE_640: variants({ XNNPACK_FP32: YOLO26_XLARGE_SEG_640_XNNPACK_FP32, - }, - }, - }, + COREML_FP16: YOLO26_XLARGE_SEG_640_COREML_FP16, + }), + }), + }), }, /** @@ -1882,10 +2237,9 @@ export const models = { * model. Extremely lightweight model evaluating continuous speech * probability chunks for live mic streaming and STT preprocessing. */ - FSMN_VAD: { - DEFAULT: FSMN_VAD_XNNPACK_FP32, + FSMN_VAD: variants({ XNNPACK_FP32: FSMN_VAD_XNNPACK_FP32, - }, + }), }, /** @@ -1897,72 +2251,116 @@ export const models = { * Voice Activity Detection. Includes multilingual and English-only (`EN`) * variants across model sizes (`TINY`, `BASE`, `SMALL`). */ + // Every size defaults to Core ML over MLX on iOS: 2.5-3.1x faster end to + // end on an iPhone 16, a third of the peak memory (MLX bf16 at `SMALL` does + // not load at all), and more accurate on the same clip. Reach for MLX_INT8 + // explicitly if you want the GPU path. WHISPER: { /** * Multilingual Whisper Tiny model. Supporting 99+ languages. High speed * speech recognition. */ - TINY: { - DEFAULT: WHISPER_TINY_XNNPACK_FP32, - XNNPACK_FP32: WHISPER_TINY_XNNPACK_FP32, - COREML_FP16: WHISPER_TINY_COREML_FP16, - MLX_BF16: WHISPER_TINY_MLX_BF16, - MLX_INT8: WHISPER_TINY_MLX_INT8, - }, + TINY: variants( + { + XNNPACK_FP32: WHISPER_TINY_XNNPACK_FP32, + COREML_FP16: WHISPER_TINY_COREML_FP16, + MLX_BF16: WHISPER_TINY_MLX_BF16, + MLX_INT8: WHISPER_TINY_MLX_INT8, + VULKAN_FP16: WHISPER_TINY_VULKAN_FP16, + VULKAN_INT8: WHISPER_TINY_VULKAN_INT8, + }, + // Core ML over MLX, see the note on WHISPER. + { ios: 'COREML_FP16' } + ), /** * Multilingual Whisper Base model. Higher accuracy across supported * languages. */ - BASE: { - DEFAULT: WHISPER_BASE_XNNPACK_FP32, - XNNPACK_FP32: WHISPER_BASE_XNNPACK_FP32, - COREML_FP16: WHISPER_BASE_COREML_FP16, - MLX_BF16: WHISPER_BASE_MLX_BF16, - MLX_INT8: WHISPER_BASE_MLX_INT8, - }, + BASE: variants( + { + XNNPACK_FP32: WHISPER_BASE_XNNPACK_FP32, + COREML_FP16: WHISPER_BASE_COREML_FP16, + MLX_BF16: WHISPER_BASE_MLX_BF16, + MLX_INT8: WHISPER_BASE_MLX_INT8, + VULKAN_FP16: WHISPER_BASE_VULKAN_FP16, + VULKAN_INT8: WHISPER_BASE_VULKAN_INT8, + }, + // Core ML over MLX, see the note on WHISPER. + { ios: 'COREML_FP16' } + ), /** * Multilingual Whisper Small model. Best accuracy for complex * multi-language audio. */ - SMALL: { - DEFAULT: WHISPER_SMALL_XNNPACK_FP32, - XNNPACK_FP32: WHISPER_SMALL_XNNPACK_FP32, - COREML_FP16: WHISPER_SMALL_COREML_FP16, - MLX_INT8: WHISPER_SMALL_MLX_INT8, - }, + SMALL: variants( + { + XNNPACK_FP32: WHISPER_SMALL_XNNPACK_FP32, + COREML_FP16: WHISPER_SMALL_COREML_FP16, + MLX_INT8: WHISPER_SMALL_MLX_INT8, + VULKAN_FP16: WHISPER_SMALL_VULKAN_FP16, + VULKAN_INT8: WHISPER_SMALL_VULKAN_INT8, + }, + // Core ML over MLX, see the note on WHISPER. + { ios: 'COREML_FP16' } + ), /** English-only optimized Whisper models (`TINY`, `BASE`, `SMALL`). */ + // The only sizes with an XNNPACK int8 export, and int8 leads fp32 for all + // but `TINY`. Greedy-decoding 250 LibriSpeech test-clean clips (31 min, + // ~4600 words) through this pipeline, int8 moves base.en 4.84% -> 5.20% + // WER and small.en 3.42% -> 3.38%: too little to outweigh halving the + // download (247 MB against 399, 448 against 1129). `TINY` keeps fp32 + // first because there int8 costs 6.08% -> 7.77%, a quarter of the + // accuracy the smallest model has left. EN: { /** * English-only Whisper Tiny model. Fast and compact for English STT. */ - TINY: { - DEFAULT: WHISPER_TINY_EN_XNNPACK_FP32, - XNNPACK_FP32: WHISPER_TINY_EN_XNNPACK_FP32, - COREML_FP16: WHISPER_TINY_EN_COREML_FP16, - MLX_BF16: WHISPER_TINY_EN_MLX_BF16, - MLX_INT8: WHISPER_TINY_EN_MLX_INT8, - }, + TINY: variants( + { + XNNPACK_FP32: WHISPER_TINY_EN_XNNPACK_FP32, + XNNPACK_INT8: WHISPER_TINY_EN_XNNPACK_INT8, + COREML_FP16: WHISPER_TINY_EN_COREML_FP16, + MLX_BF16: WHISPER_TINY_EN_MLX_BF16, + MLX_INT8: WHISPER_TINY_EN_MLX_INT8, + VULKAN_FP16: WHISPER_TINY_EN_VULKAN_FP16, + VULKAN_INT8: WHISPER_TINY_EN_VULKAN_INT8, + }, + // Core ML over MLX, see the note on WHISPER. + { ios: 'COREML_FP16' } + ), /** * English-only Whisper Base model. High accuracy English speech * recognition. */ - BASE: { - DEFAULT: WHISPER_BASE_EN_XNNPACK_FP32, - XNNPACK_FP32: WHISPER_BASE_EN_XNNPACK_FP32, - COREML_FP16: WHISPER_BASE_EN_COREML_FP16, - MLX_BF16: WHISPER_BASE_EN_MLX_BF16, - MLX_INT8: WHISPER_BASE_EN_MLX_INT8, - }, + BASE: variants( + { + XNNPACK_INT8: WHISPER_BASE_EN_XNNPACK_INT8, + XNNPACK_FP32: WHISPER_BASE_EN_XNNPACK_FP32, + COREML_FP16: WHISPER_BASE_EN_COREML_FP16, + MLX_BF16: WHISPER_BASE_EN_MLX_BF16, + MLX_INT8: WHISPER_BASE_EN_MLX_INT8, + VULKAN_FP16: WHISPER_BASE_EN_VULKAN_FP16, + VULKAN_INT8: WHISPER_BASE_EN_VULKAN_INT8, + }, + // Core ML over MLX, see the note on WHISPER. + { ios: 'COREML_FP16' } + ), /** * English-only Whisper Small model. Superior accuracy for English * transcription. */ - SMALL: { - DEFAULT: WHISPER_SMALL_EN_XNNPACK_FP32, - XNNPACK_FP32: WHISPER_SMALL_EN_XNNPACK_FP32, - COREML_FP16: WHISPER_SMALL_EN_COREML_FP16, - MLX_INT8: WHISPER_SMALL_EN_MLX_INT8, - }, + SMALL: variants( + { + XNNPACK_INT8: WHISPER_SMALL_EN_XNNPACK_INT8, + XNNPACK_FP32: WHISPER_SMALL_EN_XNNPACK_FP32, + COREML_FP16: WHISPER_SMALL_EN_COREML_FP16, + MLX_INT8: WHISPER_SMALL_EN_MLX_INT8, + VULKAN_FP16: WHISPER_SMALL_EN_VULKAN_FP16, + VULKAN_INT8: WHISPER_SMALL_EN_VULKAN_INT8, + }, + // Core ML over MLX, see the note on WHISPER. + { ios: 'COREML_FP16' } + ), }, }, }, @@ -1987,58 +2385,55 @@ export const models = { * reasoning, instruction following, and fast multi-turn conversational chat * on mobile devices. */ - LFM2_5_1_2B: { - DEFAULT: LFM2_5_1_2B_XNNPACK_8DA4W, + LFM2_5_1_2B: variants({ XNNPACK_8DA4W: LFM2_5_1_2B_XNNPACK_8DA4W, XNNPACK_FP16: LFM2_5_1_2B_XNNPACK_FP16, MLX_INT4: LFM2_5_1_2B_MLX_INT4, - }, + }), /** * Liquid AI LFM 2.5 350M ultra-compact hybrid language model. Optimized for * minimal memory footprint and sub-second first-token response times. Ideal * for lightweight text completion, fast intent classification, query * routing, and low-latency chat on resource-constrained edge hardware. */ - LFM2_5_350M: { - DEFAULT: LFM2_5_350M_XNNPACK_8DA4W, + LFM2_5_350M: variants({ XNNPACK_8DA4W: LFM2_5_350M_XNNPACK_8DA4W, XNNPACK_FP16: LFM2_5_350M_XNNPACK_FP16, MLX_INT4: LFM2_5_350M_MLX_INT4, - }, + }), /** * Liquid AI LFM 2.5 VL 450M lightweight multimodal vision-language model. * Combines Liquid hybrid language modeling with visual token embeddings for * real-time on-device visual question answering (VQA), image description, * UI element inspection, and low-latency multimodal conversational agents. */ - LFM2_5_VL_450M: { - DEFAULT: LFM2_5_VL_450M_XNNPACK_8DA4W, + LFM2_5_VL_450M: variants({ XNNPACK_8DA4W: LFM2_5_VL_450M_XNNPACK_8DA4W, MLX_INT4: LFM2_5_VL_450M_MLX_INT4, VULKAN_8DA4W: LFM2_5_VL_450M_VULKAN_8DA4W, - }, + }), /** * Liquid AI LFM 2.5 VL 1.6B high-capacity vision-language model. Provides * fine-grained visual scene understanding, document/chart interpretation, * detailed image captioning, and multi-turn visual dialogue with higher * precision and reasoning fidelity than the 450M variant. */ - LFM2_5_VL_1_6B: { - DEFAULT: LFM2_5_VL_1_6B_XNNPACK_8DA4W, + LFM2_5_VL_1_6B: variants({ XNNPACK_8DA4W: LFM2_5_VL_1_6B_XNNPACK_8DA4W, VULKAN_8DA4W: LFM2_5_VL_1_6B_VULKAN_8DA4W, - }, + MLX_INT4: LFM2_5_VL_1_6B_MLX_INT4, + MLX_INT8: LFM2_5_VL_1_6B_MLX_INT8, + }), /** * Bielik v3 1.5B bilingual Polish & English language model, developed by * SpeakLeash. Fine-tuned on curated Polish corpora and instruction datasets * for native Polish cultural nuance, grammar accuracy, idioms, and * high-fidelity bidirectional Polish-English translation. */ - BIELIK_V3_1_5B: { - DEFAULT: BIELIK_V3_1_5B_XNNPACK_8DA4W, + BIELIK_V3_1_5B: variants({ XNNPACK_8DA4W: BIELIK_V3_1_5B_XNNPACK_8DA4W, XNNPACK_FP16: BIELIK_V3_1_5B_XNNPACK_FP16, - }, + }), /** * Meta Llama 3.2 1B lightweight instruction-tuned multilingual model. * Features Grouped-Query Attention (GQA) and SpinQuant quantization for @@ -2046,169 +2441,154 @@ export const models = { * text summarization, prompt rewriting, and lightweight conversational * assistance. */ - LLAMA3_2_1B: { - DEFAULT: LLAMA3_2_1B_SPINQUANT, + LLAMA3_2_1B: variants({ XNNPACK_SPINQUANT: LLAMA3_2_1B_SPINQUANT, XNNPACK_BF16: LLAMA3_2_1B_BF16, - }, + }), /** * Meta Llama 3.2 3B instruction-tuned multilingual language model. Delivers * strong instruction adherence, multi-turn reasoning, and high-quality * content creation across 8+ core languages while maintaining a compact * on-device memory profile. */ - LLAMA3_2_3B: { - DEFAULT: LLAMA3_2_3B_SPINQUANT, + LLAMA3_2_3B: variants({ XNNPACK_SPINQUANT: LLAMA3_2_3B_SPINQUANT, XNNPACK_BF16: LLAMA3_2_3B_BF16, - }, + }), /** * Hugging Face SmolLM2 135M ultra-compact language model. Engineered for * micro-memory footprints, instant token generation, text classification, * and background processing on low-power devices. */ - SMOLLM2_135M: { - DEFAULT: SMOLLM2_135M_8DA8W, + SMOLLM2_135M: variants({ XNNPACK_8DA8W: SMOLLM2_135M_8DA8W, - }, + }), /** * Hugging Face SmolLM2 360M compact instruction-tuned model. Provides a * practical balance between fast mobile generation speed and conversational * coherence, ideal for lightweight on-device assistants, text * simplification, and structured data extraction. */ - SMOLLM2_360M: { - DEFAULT: SMOLLM2_360M_8DA8W, + SMOLLM2_360M: variants({ XNNPACK_8DA8W: SMOLLM2_360M_8DA8W, - }, + }), /** * Hugging Face SmolLM2 1.7B language model trained on curated educational, * synthetic, and web data. Delivers competitive reasoning, creative text * generation, and general knowledge Q&A performance approaching larger * 2B-3B models while maintaining fast on-device inference. */ - SMOLLM2_1_7B: { - DEFAULT: SMOLLM2_1_7B_8DA8W, + SMOLLM2_1_7B: variants({ XNNPACK_8DA8W: SMOLLM2_1_7B_8DA8W, - }, + }), /** * Hammer 2.1 0.5B specialized function-calling model. Fine-tuned * specifically for agentic tool use, structured JSON extraction, and * single/multi-tool invocation with ultra-low latency for real-time mobile * tool calling flows. */ - HAMMER2_1_0_5B: { - DEFAULT: HAMMER2_1_0_5B_XNNPACK_8DA4W, + HAMMER2_1_0_5B: variants({ XNNPACK_8DA4W: HAMMER2_1_0_5B_XNNPACK_8DA4W, XNNPACK_BF16: HAMMER2_1_0_5B_XNNPACK_BF16, - }, + }), /** * Hammer 2.1 1.5B function-calling language model. Optimized for multi-tool * agentic workflows, API parameter schema validation, and structured JSON * output generation on edge devices. */ - HAMMER2_1_1_5B: { - DEFAULT: HAMMER2_1_1_5B_XNNPACK_8DA4W, + HAMMER2_1_1_5B: variants({ XNNPACK_8DA4W: HAMMER2_1_1_5B_XNNPACK_8DA4W, XNNPACK_BF16: HAMMER2_1_1_5B_XNNPACK_BF16, - }, + }), /** * Hammer 2.1 3B high-capacity function-calling model. Provides top-tier * tool selection precision, multi-turn tool calling, error recovery, and * strict compliance with complex TypeScript/JSON schema specifications in * autonomous mobile agent pipelines. */ - HAMMER2_1_3B: { - DEFAULT: HAMMER2_1_3B_XNNPACK_8DA4W, + HAMMER2_1_3B: variants({ XNNPACK_8DA4W: HAMMER2_1_3B_XNNPACK_8DA4W, XNNPACK_BF16: HAMMER2_1_3B_XNNPACK_BF16, - }, + }), /** * Microsoft Phi-4 Mini 3.8B high-density reasoning model. Trained on * synthetic textbook-grade datasets for state-of-the-art on-device STEM * problem solving, complex mathematical reasoning, multi-step code * synthesis, and structured analytical tasks. */ - PHI4_MINI: { - DEFAULT: PHI4_MINI_XNNPACK_8DA4W, + PHI4_MINI: variants({ XNNPACK_8DA4W: PHI4_MINI_XNNPACK_8DA4W, XNNPACK_BF16: PHI4_MINI_XNNPACK_BF16, - }, + }), /** * Alibaba Qwen 2.5 0.5B ultra-lightweight multilingual model. Trained on * 18T tokens supporting 29+ languages; optimized for near-instant response * times, basic instruction following, multilingual translation, and * lightweight conversational assistants on mobile devices. */ - QWEN2_5_0_5B: { - DEFAULT: QWEN2_5_0_5B_XNNPACK_8DA4W, + QWEN2_5_0_5B: variants({ XNNPACK_8DA4W: QWEN2_5_0_5B_XNNPACK_8DA4W, XNNPACK_BF16: QWEN2_5_0_5B_XNNPACK_BF16, - }, + }), /** * Alibaba Qwen 2.5 1.5B multilingual instruction model. Combines broad * multilingual comprehension across 29+ languages with strong coding and * math capabilities, well suited for interactive chat, summarization, and * cross-lingual translation. */ - QWEN2_5_1_5B: { - DEFAULT: QWEN2_5_1_5B_XNNPACK_8DA4W, + QWEN2_5_1_5B: variants({ XNNPACK_8DA4W: QWEN2_5_1_5B_XNNPACK_8DA4W, XNNPACK_BF16: QWEN2_5_1_5B_XNNPACK_BF16, - }, + }), /** * Alibaba Qwen 2.5 3B high-capability multilingual model. Delivers strong * reasoning, coding, mathematics, and multilingual fluency across 29+ * languages for in-depth text generation and complex multi-turn dialogue. */ - QWEN2_5_3B: { - DEFAULT: QWEN2_5_3B_XNNPACK_8DA4W, + QWEN2_5_3B: variants({ XNNPACK_8DA4W: QWEN2_5_3B_XNNPACK_8DA4W, XNNPACK_BF16: QWEN2_5_3B_XNNPACK_BF16, - }, + }), /** * Alibaba Qwen 3 0.6B next-generation compact language model. Features * updated architectural optimizations for reduced latency, enhanced * multilingual token representation, and efficient conversational * turn-taking on mobile devices. */ - QWEN3_0_6B: { - DEFAULT: QWEN3_0_6B_XNNPACK_8DA4W, + QWEN3_0_6B: variants({ XNNPACK_8DA4W: QWEN3_0_6B_XNNPACK_8DA4W, XNNPACK_BF16: QWEN3_0_6B_XNNPACK_BF16, - }, + }), /** * Alibaba Qwen 3 1.7B next-generation multilingual language model. Balances * high reasoning capability, general knowledge retrieval, coding * proficiency, and conversational fluidity across multiple languages. */ - QWEN3_1_7B: { - DEFAULT: QWEN3_1_7B_XNNPACK_8DA4W, + QWEN3_1_7B: variants({ XNNPACK_8DA4W: QWEN3_1_7B_XNNPACK_8DA4W, XNNPACK_BF16: QWEN3_1_7B_XNNPACK_BF16, - }, + }), /** * Alibaba Qwen 3 4B high-capacity generative model. Delivers advanced * multi-step reasoning, comprehensive world knowledge, complex coding * capabilities, and top-tier multilingual performance for demanding * on-device AI applications. */ - QWEN3_4B: { - DEFAULT: QWEN3_4B_XNNPACK_8DA4W, + QWEN3_4B: variants({ XNNPACK_8DA4W: QWEN3_4B_XNNPACK_8DA4W, XNNPACK_BF16: QWEN3_4B_XNNPACK_BF16, - }, + }), /** * Google Gemma 4 E2B generative language model. Built on Google's Gemini * research and architecture innovations, offering high-fidelity instruction * following, creative text generation, and reasoning efficiency optimized * for mobile deployment. */ - GEMMA4_E2B: { - DEFAULT: GEMMA4_E2B_XNNPACK_8DA4W, + GEMMA4_E2B: variants({ XNNPACK_8DA4W: GEMMA4_E2B_XNNPACK_8DA4W, + VULKAN_8DA4W: GEMMA4_E2B_VULKAN_8DA4W, MLX_INT4: GEMMA4_E2B_MLX_INT4, - }, + }), }, /** @@ -2221,71 +2601,86 @@ export const models = { * vector space. Optimized for fast, general-purpose semantic search, * sentence similarity, and clustering. */ - ALL_MINILM_L6_V2: { - DEFAULT: ALL_MINILM_L6_V2_EMBEDDINGS, + ALL_MINILM_L6_V2: variants({ XNNPACK_FP32: ALL_MINILM_L6_V2_EMBEDDINGS, - }, + COREML_FP16: ALL_MINILM_L6_V2_COREML_FP16, + VULKAN_FP16: ALL_MINILM_L6_V2_VULKAN_FP16, + }), /** * High-quality 768-dimensional sentence transformer model based on MPNet. * Provides higher quality semantic embeddings compared to MiniLM. */ - ALL_MPNET_BASE_V2: { - DEFAULT: ALL_MPNET_BASE_V2_EMBEDDINGS, + ALL_MPNET_BASE_V2: variants({ XNNPACK_FP32: ALL_MPNET_BASE_V2_EMBEDDINGS, - }, + VULKAN_FP16: ALL_MPNET_BASE_V2_VULKAN_FP16, + VULKAN_INT8: ALL_MPNET_BASE_V2_VULKAN_INT8, + }), /** * 384-dimensional sentence transformer fine-tuned specifically for semantic * QA matching using cosine similarity. */ - MULTI_QA_MINILM_L6_COS_V1: { - DEFAULT: MULTI_QA_MINILM_L6_COS_V1_EMBEDDINGS, + MULTI_QA_MINILM_L6_COS_V1: variants({ XNNPACK_FP32: MULTI_QA_MINILM_L6_COS_V1_EMBEDDINGS, - }, + COREML_FP16: MULTI_QA_MINILM_L6_COS_V1_COREML_FP16, + VULKAN_FP16: MULTI_QA_MINILM_L6_COS_V1_VULKAN_FP16, + }), /** * 768-dimensional sentence transformer fine-tuned specifically for * question-answering matching using dot product distance. */ - MULTI_QA_MPNET_BASE_DOT_V1: { - DEFAULT: MULTI_QA_MPNET_BASE_DOT_V1_EMBEDDINGS, + MULTI_QA_MPNET_BASE_DOT_V1: variants({ XNNPACK_FP32: MULTI_QA_MPNET_BASE_DOT_V1_EMBEDDINGS, - }, + VULKAN_FP16: MULTI_QA_MPNET_BASE_DOT_V1_VULKAN_FP16, + VULKAN_INT8: MULTI_QA_MPNET_BASE_DOT_V1_VULKAN_INT8, + }), /** * 384-dimensional sentence transformer supporting 50+ languages for * cross-lingual semantic similarity. */ - PARAPHRASE_MULTILINGUAL_MINILM_L12_V2: { - DEFAULT: PARAPHRASE_MULTILINGUAL_MINILM_L12_V2_EMBEDDINGS, + PARAPHRASE_MULTILINGUAL_MINILM_L12_V2: variants({ XNNPACK_8DA4W: PARAPHRASE_MULTILINGUAL_MINILM_L12_V2_EMBEDDINGS, - }, + XNNPACK_FP32: PARAPHRASE_MULTILINGUAL_MINILM_L12_V2_XNNPACK_FP32, + COREML_FP16: PARAPHRASE_MULTILINGUAL_MINILM_L12_V2_COREML_FP16, + VULKAN_FP16: PARAPHRASE_MULTILINGUAL_MINILM_L12_V2_VULKAN_FP16, + }), /** * Multilingual sentence transformer supporting 50+ languages, based on * distilled Universal Sentence Encoder (512-dim output). */ - DISTILUSE_BASE_MULTILINGUAL_CASED_V2: { - DEFAULT: DISTILUSE_BASE_MULTILINGUAL_CASED_V2_EMBEDDINGS, - XNNPACK_8DA4W: DISTILUSE_BASE_MULTILINGUAL_CASED_V2_EMBEDDINGS, - MLX_INT8: DISTILUSE_BASE_MULTILINGUAL_CASED_V2_MLX_INT8, - }, + DISTILUSE_BASE_MULTILINGUAL_CASED_V2: variants( + { + XNNPACK_8DA4W: DISTILUSE_BASE_MULTILINGUAL_CASED_V2_EMBEDDINGS, + XNNPACK_FP32: DISTILUSE_BASE_MULTILINGUAL_CASED_V2_XNNPACK_FP32, + COREML_FP16: DISTILUSE_BASE_MULTILINGUAL_CASED_V2_COREML_FP16, + MLX_INT8: DISTILUSE_BASE_MULTILINGUAL_CASED_V2_MLX_INT8, + VULKAN_FP16: DISTILUSE_BASE_MULTILINGUAL_CASED_V2_VULKAN_FP16, + }, + // The one Core ML/MLX pair that does not go Core ML's way: on an + // iPhone 16, warm, MLX int8 embeds in 3.46 ms against Core ML fp16's + // 3.96, at half the download and no first-use compile, and the two match + // the XNNPACK reference equally well. + { ios: 'MLX_INT8' } + ), /** * CLIP text encoder (ViT-B/32) mapping text queries into a 512-dimensional * joint text-image embedding space. Used in combination with * `imageEmbeddings.CLIP_VIT_BASE_PATCH32` for zero-shot text-to-image * search. */ - CLIP_VIT_BASE_PATCH32_TEXT: { - DEFAULT: CLIP_VIT_BASE_PATCH32_TEXT_EMBEDDINGS, + CLIP_VIT_BASE_PATCH32_TEXT: variants({ XNNPACK_FP32: CLIP_VIT_BASE_PATCH32_TEXT_EMBEDDINGS, - }, + COREML_FP16: CLIP_VIT_BASE_PATCH32_TEXT_COREML_FP16, + VULKAN_FP16: CLIP_VIT_BASE_PATCH32_TEXT_VULKAN_FP16, + }), /** * Liquid AI LFM 2.5 350M parameter embedding model for asymmetric search * and retrieval tasks. Prompts queries with `query: ` (the default) and * passages with `document: ` via {@link TextEmbedder.embed}. */ - LFM2_5_EMBEDDING_350M: { - DEFAULT: LFM2_5_EMBEDDING_350M_EMBEDDINGS, + LFM2_5_EMBEDDING_350M: variants({ XNNPACK_8DA4W: LFM2_5_EMBEDDING_350M_EMBEDDINGS, MLX_INT4: LFM2_5_EMBEDDING_350M_MLX_INT4, - }, + }), }, /** @@ -2298,20 +2693,18 @@ export const models = { * OpenAI-style detector covering 8 common PII types (name, email, phone, * address, and similar). Compact label space, best for general redaction. */ - OPENAI: { - DEFAULT: PRIVACY_FILTER_OPENAI_XNNPACK_8DA4W, + OPENAI: variants({ XNNPACK_8DA4W: PRIVACY_FILTER_OPENAI_XNNPACK_8DA4W, MLX_INT4: PRIVACY_FILTER_OPENAI_MLX_INT4, - }, + }), /** * Nemotron-based detector covering 55 fine-grained PII types. Larger label * space for stricter compliance-oriented redaction. */ - NEMOTRON: { - DEFAULT: PRIVACY_FILTER_NEMOTRON_XNNPACK_8DA4W, + NEMOTRON: variants({ XNNPACK_8DA4W: PRIVACY_FILTER_NEMOTRON_XNNPACK_8DA4W, MLX_INT8: PRIVACY_FILTER_NEMOTRON_MLX_INT8, - }, + }), }, /** @@ -2323,12 +2716,17 @@ export const models = { * shared text-image space. Used for zero-shot visual classification and * cross-modal image search. */ - CLIP_VIT_BASE_PATCH32: { - DEFAULT: CLIP_VIT_BASE_PATCH32_IMAGE_XNNPACK_FP32, - XNNPACK_FP32: CLIP_VIT_BASE_PATCH32_IMAGE_XNNPACK_FP32, - COREML_FP16: CLIP_VIT_BASE_PATCH32_IMAGE_COREML_FP16, - MLX_INT8: CLIP_VIT_BASE_PATCH32_IMAGE_MLX_INT8, - }, + CLIP_VIT_BASE_PATCH32: variants( + { + XNNPACK_FP32: CLIP_VIT_BASE_PATCH32_IMAGE_XNNPACK_FP32, + COREML_FP16: CLIP_VIT_BASE_PATCH32_IMAGE_COREML_FP16, + MLX_INT8: CLIP_VIT_BASE_PATCH32_IMAGE_MLX_INT8, + VULKAN_FP16: CLIP_VIT_BASE_PATCH32_IMAGE_VULKAN_FP16, + }, + // Core ML over MLX: 3.5 ms against 14.1 on an iPhone 16, the widest + // margin of any pair that ships both. + { ios: 'COREML_FP16' } + ), }, /** @@ -2340,11 +2738,10 @@ export const models = { * generation model based on DreamShaper. Generates high-quality images from * text prompts in real time. */ - SDXS_512_DREAMSHAPER: { - DEFAULT: SDXS_512_DREAMSHAPER_XNNPACK_FP32, + SDXS_512_DREAMSHAPER: variants({ XNNPACK_FP32: SDXS_512_DREAMSHAPER_XNNPACK_FP32, COREML_FP16: SDXS_512_DREAMSHAPER_COREML_FP16, - }, + }), }, /** @@ -2357,63 +2754,59 @@ export const models = { * natural, highly expressive speech synthesis with configurable speaker * voice presets (see {@link SUPERTONIC_DEFAULT_VOICE_NAMES}). */ - SUPERTONIC: { - DEFAULT: SUPERTONIC_3_XNNPACK_FP32, + SUPERTONIC: variants({ XNNPACK_FP32: SUPERTONIC_3_XNNPACK_FP32, MLX_FP32: SUPERTONIC_3_MLX_FP32, - }, + VULKAN_FP16: SUPERTONIC_3_VULKAN_FP16, + }), /** * Kokoro — a lightweight phoneme-driven Text-to-Speech model. Each language * entry bundles the matching model weights, grapheme-to-phoneme assets and * the voices available for that language, nested per backend. */ + // The Core ML builds are the registry's only fp32 Core ML exports, which + // keeps them off the fp16-only Neural Engine, and they still default on + // iOS: the duration predictor runs 14-21x faster warm than the XNNPACK one + // on an iPhone 16. The cost is a one-time ~13s compile on first use, cached + // across launches; reach for `XNNPACK_FP32` explicitly to avoid it. KOKORO: { - EN_US: { - DEFAULT: KOKORO_EN_US_XNNPACK_FP32, + EN_US: variants({ XNNPACK_FP32: KOKORO_EN_US_XNNPACK_FP32, COREML_FP32: KOKORO_EN_US_COREML_FP32, - }, - EN_GB: { - DEFAULT: KOKORO_EN_GB_XNNPACK_FP32, + }), + EN_GB: variants({ XNNPACK_FP32: KOKORO_EN_GB_XNNPACK_FP32, COREML_FP32: KOKORO_EN_GB_COREML_FP32, - }, - ES: { - DEFAULT: KOKORO_ES_XNNPACK_FP32, + }), + ES: variants({ XNNPACK_FP32: KOKORO_ES_XNNPACK_FP32, COREML_FP32: KOKORO_ES_COREML_FP32, - }, - FR: { - DEFAULT: KOKORO_FR_XNNPACK_FP32, + }), + FR: variants({ XNNPACK_FP32: KOKORO_FR_XNNPACK_FP32, COREML_FP32: KOKORO_FR_COREML_FP32, - }, - IT: { - DEFAULT: KOKORO_IT_XNNPACK_FP32, + }), + IT: variants({ XNNPACK_FP32: KOKORO_IT_XNNPACK_FP32, COREML_FP32: KOKORO_IT_COREML_FP32, - }, - PT: { - DEFAULT: KOKORO_PT_XNNPACK_FP32, + }), + PT: variants({ XNNPACK_FP32: KOKORO_PT_XNNPACK_FP32, COREML_FP32: KOKORO_PT_COREML_FP32, - }, - HI: { - DEFAULT: KOKORO_HI_XNNPACK_FP32, + }), + HI: variants({ XNNPACK_FP32: KOKORO_HI_XNNPACK_FP32, COREML_FP32: KOKORO_HI_COREML_FP32, - }, - PL: { - DEFAULT: KOKORO_PL_XNNPACK_FP32, + }), + PL: variants({ XNNPACK_FP32: KOKORO_PL_XNNPACK_FP32, COREML_FP32: KOKORO_PL_COREML_FP32, - }, - DE: { - DEFAULT: KOKORO_DE_XNNPACK_FP32, + }), + DE: variants({ XNNPACK_FP32: KOKORO_DE_XNNPACK_FP32, COREML_FP32: KOKORO_DE_COREML_FP32, - }, + }), }, }, @@ -2432,12 +2825,12 @@ export const models = { /** * PP-OCRv6 Small multilingual OCR model. */ - PPOCRV6_SMALL: { - DEFAULT: PPOCRV6_SMALL_XNNPACK_INT8, + PPOCRV6_SMALL: variants({ XNNPACK: PPOCRV6_SMALL_XNNPACK_INT8, + XNNPACK_FP32: PPOCRV6_SMALL_XNNPACK_FP32, COREML: PPOCRV6_SMALL_COREML_INT8, VULKAN: PPOCRV6_SMALL_VULKAN_FP16, - }, + }), }, }, };