From e7c3fbe5665b6295fdd12baae392158595886497 Mon Sep 17 00:00:00 2001 From: Laurids Kern Date: Tue, 20 Oct 2026 13:39:55 +0200 Subject: [PATCH 01/63] feat(storage): inspect and reset app data Read MMKV, AsyncStorage, UserDefaults, SharedPreferences, DataStore and SQLite stores from the app's data directory without app code, mark keys changed since the first read, and clear one store or all app data. --- README.md | 5 +- docs/telemetry.md | 13 + skills/mobile-dev/SKILL.md | 2 + src/server/app-storage/container.ts | 170 ++++++++++++ src/server/app-storage/formats.ts | 245 ++++++++++++++++++ src/server/app-storage/mmkv.ts | 121 +++++++++ src/server/app-storage/reader.ts | 158 +++++++++++ src/server/app-storage/tools.ts | 85 ++++++ src/server/plugin.ts | 2 + src/server/telemetry.ts | 9 +- src/shared/app-storage.ts | 59 +++++ src/shared/telemetry.ts | 2 +- src/ui/components/storage-view.tsx | 237 +++++++++++++++++ src/ui/components/workspace.tsx | 35 ++- src/ui/storage-panel.ts | 204 +++++++++++++++ src/ui/workspace-app.tsx | 6 +- tests/app-storage.test.ts | 241 +++++++++++++++++ tests/fixtures/app-storage/mmkv/expiring | Bin 0 -> 37 bytes tests/fixtures/app-storage/mmkv/expiring.crc | Bin 0 -> 112 bytes tests/fixtures/app-storage/mmkv/fixture | Bin 0 -> 169 bytes tests/fixtures/app-storage/mmkv/fixture.crc | Bin 0 -> 112 bytes tests/fixtures/app-storage/mmkv/locked | Bin 0 -> 22 bytes tests/fixtures/app-storage/mmkv/locked.crc | Bin 0 -> 112 bytes tests/fixtures/app-storage/mmkv/mmkv.default | Bin 0 -> 26 bytes .../app-storage/mmkv/mmkv.default.crc | Bin 0 -> 112 bytes tests/react-ui.test.ts | 35 ++- 26 files changed, 1611 insertions(+), 18 deletions(-) create mode 100644 src/server/app-storage/container.ts create mode 100644 src/server/app-storage/formats.ts create mode 100644 src/server/app-storage/mmkv.ts create mode 100644 src/server/app-storage/reader.ts create mode 100644 src/server/app-storage/tools.ts create mode 100644 src/shared/app-storage.ts create mode 100644 src/ui/components/storage-view.tsx create mode 100644 src/ui/storage-panel.ts create mode 100644 tests/app-storage.test.ts create mode 100644 tests/fixtures/app-storage/mmkv/expiring create mode 100644 tests/fixtures/app-storage/mmkv/expiring.crc create mode 100644 tests/fixtures/app-storage/mmkv/fixture create mode 100644 tests/fixtures/app-storage/mmkv/fixture.crc create mode 100644 tests/fixtures/app-storage/mmkv/locked create mode 100644 tests/fixtures/app-storage/mmkv/locked.crc create mode 100644 tests/fixtures/app-storage/mmkv/mmkv.default create mode 100644 tests/fixtures/app-storage/mmkv/mmkv.default.crc diff --git a/README.md b/README.md index 6ce0cf5f..85330914 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,9 @@ Track app CPU and memory, expand individual thread charts, and view device-wide ![](/img/screenshot_performance.png) +### App storage +See what an app has saved without adding code to it: MMKV, AsyncStorage, UserDefaults, SharedPreferences, DataStore, SQLite databases, and the largest files in its data directory. Keys added or changed since the first read are marked, so you can watch stale flags and cached logins change. Clear one store or reset all app data. Works on iOS simulators and debuggable Android builds; MMKV value types are guessed from the stored bytes, and encrypted MMKV instances show as locked. + ## Requirements @@ -87,7 +90,7 @@ Ask Codex to work with your app, for example: - “Record CPU and memory for 30 seconds while I scroll checkout.” - “Compare these two performance recordings.” -Open Logs or Performance to inspect the selected device. Use Screenshot to attach +Open Logs, Performance, or Storage to inspect the selected device. Use Screenshot to attach an image to your next message, or Select to annotate an element or region with an instruction. Physical iOS supports taps and drags; keyboard and hardware-button controls are unavailable. diff --git a/docs/telemetry.md b/docs/telemetry.md index b223bbd5..5ee0127b 100644 --- a/docs/telemetry.md +++ b/docs/telemetry.md @@ -211,6 +211,19 @@ milliseconds with bounded aggregate windows, and counts `ui.app_tools.failure`. Results that finish after a surface or device change are not recorded. Recent URLs and the last picked app stay in the panel's local storage and are never sent. +The Storage tool (`mobile_app_storage`, `mobile_clear_app_storage`) uses the `storage` +surface. The server records `app_storage.operation.duration` in milliseconds and +`app_storage.operation.samples` per operation (`read`, `clear`, `reset`) with the device +platform and a success or failure outcome. The duration covers listing the data +directory, reading and decoding every recognized store, and for clears stopping the +app first. Unexpected failures report under `app_storage.` with a fixed +message; a release Android build, a missing app, or a stopped device counts only +through `server.error.outcome`. The panel records `ui.storage.list_apps`, `.read`, +`.clear`, `.reset`, and `.launch` round trips, `ui.storage.process` for validating +a snapshot and marking changes, gauges `ui.storage.stores` and `ui.storage.entries`, +and counts `ui.storage.failure`. Bundle IDs, store names, paths, keys, values, and +file names are never sent. + Since 0.1.152, `mobile_app_language` and `mobile_set_app_language` use the same app control coverage under `app_control.language` and `app_control.set_language`. Language tags are not sent. The device settings tools report unexpected Baguette or diff --git a/skills/mobile-dev/SKILL.md b/skills/mobile-dev/SKILL.md index 87d2eca2..286a61f1 100644 --- a/skills/mobile-dev/SKILL.md +++ b/skills/mobile-dev/SKILL.md @@ -39,6 +39,8 @@ These tools work on iOS simulators and Android devices with `{ platform, deviceI - `mobile_set_app_permission` allows, denies, or resets (`ask`) a permission so system prompts do not interrupt a flow; `permission: "all"` with `state: "ask"` restores first-launch prompts. iOS simulators cannot change camera or notification access, and changes can restart the app. - `mobile_send_push` sends an APNs payload to an app on an iOS simulator. Android push needs Firebase Cloud Messaging and is not supported. - `mobile_set_app_language` runs one app in another language and region, such as `de`, `pt-BR`, or `ar` for right-to-left layout, then relaunches it. `language: null` returns the app to the device language. The device language does not change. Android needs version 13 or later. +- `mobile_app_storage` reads what an app saved, without changes to the app: MMKV, AsyncStorage, UserDefaults, SharedPreferences, DataStore, SQLite tables, and the largest files. Use it for bugs that may come from stale state, such as a cached login or an old onboarding flag. MMKV types are guessed from bytes; encrypted MMKV stays locked. Android needs a debuggable build. +- `mobile_clear_app_storage` stops the app, then clears one store ID from `mobile_app_storage`, or all app data when `store` is omitted. It does not relaunch; call `mobile_launch_app` afterwards. The panel's settings button opens the same controls under its App tab. Its Device tab also switches to a clean status bar (9:41, full battery and signal) for screenshots and simulates LTE, 3G, Edge, or offline networks. Those device settings are panel-only, so ask the user to set them. Android throttling applies right away to standard app networking such as fetch and OkHttp, leaves local dev servers like Metro at full speed, and is unavailable while the device uses its own proxy. iOS throttling applies to URLSession and fetch requests in apps launched after it was turned on, so relaunch the app after the user enables it. diff --git a/src/server/app-storage/container.ts b/src/server/app-storage/container.ts new file mode 100644 index 00000000..cf1ea700 --- /dev/null +++ b/src/server/app-storage/container.ts @@ -0,0 +1,170 @@ +import { mkdir, readdir, readFile, rm, stat, writeFile } from "node:fs/promises"; +import { join, relative, sep } from "node:path"; +import { ExpectedOperationError } from "../../shared/error-reporting.ts"; +import { commandError, commandOutput } from "../app-control/commands.ts"; +import type { Commands } from "../app-control/commands.ts"; + +export type ContainerFile = { path: string; bytes: number }; + +// One app's data directory. Paths are relative to it and use forward slashes. +export interface AppContainer { + readonly platform: "ios" | "android"; + list(): Promise; + read(path: string): Promise; + // Copies files into a local directory under their base names, for tools such as sqlite3. + copy(paths: string[], directory: string): Promise; + remove(paths: string[]): Promise; + // The current UserDefaults domain as plist XML, including values cfprefsd has not written yet. + preferences?(path: string): Promise; + forgetPreferences?(path: string): Promise; + // Removes everything the app stored, leaving it installed. + reset(): Promise; +} + +// A container directory can hold caches with many files; listing stops here. +export const maxContainerFiles = 20000; + +const shellQuote = (value: string) => `'${value.replaceAll("'", "'\\''")}'`; + +function safePath(path: string) { + if (path === "" || path.startsWith("/") || path.split("/").some(part => part === ".." || part === "")) throw new ExpectedOperationError("invalid_input", "The storage path is outside the app's data."); + return path; +} + +export class SimulatorContainer implements AppContainer { + readonly platform = "ios"; + private readonly root: string; + private readonly deviceId: string; + private readonly commands: Commands; + + constructor(root: string, deviceId: string, commands: Commands) { + this.root = root; + this.deviceId = deviceId; + this.commands = commands; + } + + static async open(deviceId: string, bundleId: string, commands: Commands) { + let output: string; + try { output = await commands.run("xcrun", ["simctl", "get_app_container", deviceId, bundleId, "data"], { timeout: 15000 }); } + catch (error) { + if (/No such file or directory|not installed/i.test(commandOutput(error))) throw new ExpectedOperationError("invalid_input", `${bundleId} is not installed on this simulator.`); + throw commandError(error, "Finding the app data"); + } + return new SimulatorContainer(output.trim(), deviceId, commands); + } + + async list() { + const files: ContainerFile[] = []; + const walk = async (directory: string) => { + const entries = await readdir(directory, { withFileTypes: true }).catch(() => []); + for (const entry of entries) { + if (files.length >= maxContainerFiles) return; + const path = join(directory, entry.name); + if (entry.isDirectory()) await walk(path); + else if (entry.isFile()) { + const info = await stat(path).catch(() => undefined); + if (info) files.push({ path: relative(this.root, path).split(sep).join("/"), bytes: info.size }); + } + } + }; + await walk(this.root); + return files; + } + + async read(path: string) { return readFile(join(this.root, safePath(path))); } + + async copy(paths: string[], directory: string) { + await Promise.all(paths.map(async path => writeFile(join(directory, path.slice(path.lastIndexOf("/") + 1)), await this.read(path)))); + } + + async remove(paths: string[]) { + await Promise.all(paths.map(path => rm(join(this.root, safePath(path)), { recursive: true, force: true }))); + } + + private defaults(args: string[]) { + return this.commands.run("xcrun", ["simctl", "spawn", this.deviceId, "defaults", ...args], { timeout: 15000, maxBuffer: 32 * 1024 * 1024 }); + } + + // `defaults` asks cfprefsd, which may hold writes the app made after the plist was last saved. + async preferences(path: string) { + const domain = join(this.root, safePath(path)).replace(/\.plist$/, ""); + try { return await this.defaults(["export", domain, "-"]); } + catch { return this.commands.run("plutil", ["-convert", "xml1", "-o", "-", `${domain}.plist`], { timeout: 10000, maxBuffer: 32 * 1024 * 1024 }); } + } + + // Deleting only the file would leave cfprefsd's cached copy, which the app would read and save again. + async forgetPreferences(path: string) { + const domain = join(this.root, safePath(path)).replace(/\.plist$/, ""); + await this.defaults(["delete", domain]).catch(() => ""); + } + + async reset() { + const preferences = (await this.list()).filter(file => /^Library\/Preferences\/[^/]+\.plist$/.test(file.path)); + for (const file of preferences) await this.forgetPreferences(file.path); + // The container keeps its top-level layout and metadata; everything the app created inside goes. + for (const directory of ["Documents", "Library", "tmp"]) { + const entries = await readdir(join(this.root, directory)).catch(() => []); + await Promise.all(entries.map(entry => rm(join(this.root, directory, entry), { recursive: true, force: true }))); + } + for (const directory of ["Documents", "Library/Caches", "Library/Preferences", "tmp"]) await mkdir(join(this.root, directory), { recursive: true }); + } +} + +export class AndroidContainer implements AppContainer { + readonly platform = "android"; + private readonly serial: string; + private readonly bundleId: string; + private readonly commands: Commands; + + constructor(serial: string, bundleId: string, commands: Commands) { + this.serial = serial; + this.bundleId = bundleId; + this.commands = commands; + } + + // run-as starts in the app's data directory with the app's user, so it only works for debuggable builds. + private async runAs(action: string, command: string, timeout = 20000) { + try { return await this.commands.run(await this.commands.adb(), ["-s", this.serial, "shell", `run-as ${this.bundleId} sh -c ${shellQuote(command)}`], { timeout, maxBuffer: 32 * 1024 * 1024 }); } + catch (error) { throw runAsError(error, action, this.bundleId); } + } + + async list() { + // toybox stat prints size and name; `find` without -print0 because older toybox lacks it. + const output = await this.runAs("Listing app data", `find . -type f -exec stat -c '%s %n' {} + 2>/dev/null | head -n ${maxContainerFiles}; true`, 30000); + if (/^run-as:/m.test(output)) throw runAsError({ stdout: output }, "Listing app data", this.bundleId); + return output.split("\n").flatMap(line => { + const match = /^(\d+) \.\/(.+)$/.exec(line.trimEnd()); + return match ? [{ path: match[2], bytes: Number(match[1]) }] : []; + }); + } + + async read(path: string) { + const adb = await this.commands.adb(); + try { return await this.commands.readBinary(adb, ["-s", this.serial, "exec-out", `run-as ${this.bundleId} cat ${shellQuote(safePath(path))}`], { timeout: 30000 }); } + catch (error) { throw runAsError(error, "Reading app data", this.bundleId); } + } + + async copy(paths: string[], directory: string) { + for (const path of paths) await writeFile(join(directory, path.slice(path.lastIndexOf("/") + 1)), await this.read(path)); + } + + async remove(paths: string[]) { + if (paths.length === 0) return; + await this.runAs("Clearing app data", `rm -rf -- ${paths.map(path => shellQuote(safePath(path))).join(" ")}`); + } + + // pm clear also stops the app and works for release builds. + async reset() { + let output: string; + try { output = await this.commands.run(await this.commands.adb(), ["-s", this.serial, "shell", `pm clear ${this.bundleId}`], { timeout: 30000 }); } + catch (error) { throw commandError(error, "Clearing app data"); } + if (!/Success/.test(output)) throw new ExpectedOperationError("invalid_input", `Clearing app data failed: ${output.trim().split("\n")[0] || "pm clear did not succeed."}`); + } +} + +export function runAsError(error: unknown, action: string, bundleId: string): Error { + const output = commandOutput(error); + if (/not debuggable/.test(output)) return new ExpectedOperationError("unsupported_platform", `${bundleId} is a release build. Android only allows reading app data of debuggable builds; Reset app data still works.`); + if (/unknown package|Unknown package|is unknown/.test(output)) return new ExpectedOperationError("invalid_input", `${bundleId} is not installed on this device.`); + return commandError(error, action); +} diff --git a/src/server/app-storage/formats.ts b/src/server/app-storage/formats.ts new file mode 100644 index 00000000..3585ca72 --- /dev/null +++ b/src/server/app-storage/formats.ts @@ -0,0 +1,245 @@ +import type { EntryType, StoreKind } from "../../shared/app-storage.ts"; +import { jsonType, readVarint } from "./mmkv.ts"; + +export type ParsedEntry = { key: string; type: EntryType; value: string; bytes: number; guessed?: boolean }; + +// ---- Property lists (UserDefaults), parsed from the XML that `defaults export` and `plutil -convert xml1` print. + +class PlistData { readonly bytes: Buffer; constructor(bytes: Buffer) { this.bytes = bytes; } } +class PlistDate { readonly iso: string; constructor(iso: string) { this.iso = iso; } } +type PlistValue = string | number | bigint | boolean | PlistData | PlistDate | PlistValue[] | { [key: string]: PlistValue }; + +const entities: Record = { lt: "<", gt: ">", amp: "&", quot: "\"", apos: "'" }; +function unescapeXml(text: string) { + return text.replace(/&(#x[0-9a-f]+|#\d+|lt|gt|amp|quot|apos);/gi, (match, name: string) => { + if (name[0] !== "#") return entities[name.toLowerCase()] ?? match; + const code = name[1] === "x" || name[1] === "X" ? Number.parseInt(name.slice(2), 16) : Number.parseInt(name.slice(1), 10); + return Number.isFinite(code) ? String.fromCodePoint(code) : match; + }); +} + +type Token = { tag: string; close: boolean; empty: boolean } | { text: string }; +function tokens(xml: string): Token[] { + const body = xml.replace(/<\?[\s\S]*?\?>||]*>/g, ""); + const result: Token[] = []; + for (const match of body.matchAll(/<(\/?)([A-Za-z]+)(?:\s[^>]*?)?(\/?)>|([^<]+)/g)) { + if (match[4] !== undefined) result.push({ text: match[4] }); + else result.push({ tag: match[2], close: match[1] === "/", empty: match[3] === "/" }); + } + return result; +} + +export function parsePlistXml(xml: string): Record { + const list = tokens(xml); + let index = 0; + const next = () => list[index++]; + const skipText = () => { while (index < list.length && "text" in list[index] && /^\s*$/.test((list[index] as { text: string }).text)) index++; }; + const textUntil = (tag: string) => { + let text = ""; + for (let token = next(); token !== undefined; token = next()) { + if ("text" in token) text += token.text; + else if (token.close && token.tag === tag) return unescapeXml(text); + } + throw new Error(`Unclosed <${tag}>.`); + }; + const value = (): PlistValue => { + skipText(); + const token = next(); + if (token === undefined || "text" in token || token.close) throw new Error("Expected a plist value."); + const { tag, empty } = token; + if (tag === "true" || tag === "false") return tag === "true"; + if (tag === "dict") { + const dict: Record = {}; + if (empty) return dict; + for (;;) { + skipText(); + const key = next(); + if (key === undefined || "text" in key) throw new Error("Expected a dict key."); + if (key.close && key.tag === "dict") return dict; + dict[key.empty ? "" : textUntil("key")] = value(); + } + } + if (tag === "array") { + const array: PlistValue[] = []; + if (empty) return array; + for (;;) { + skipText(); + const peek = list[index]; + if (peek && !("text" in peek) && peek.close && peek.tag === "array") { index++; return array; } + array.push(value()); + } + } + const text = empty ? "" : textUntil(tag); + if (tag === "string") return text; + if (tag === "integer") return Number.isSafeInteger(Number(text)) ? Number(text) : BigInt(text.trim()); + if (tag === "real") return Number(text); + if (tag === "date") return new PlistDate(text.trim()); + if (tag === "data") return new PlistData(Buffer.from(text.replace(/\s+/g, ""), "base64")); + throw new Error(`Unsupported plist element <${tag}>.`); + }; + skipText(); + const root = next(); + if (root === undefined || "text" in root || root.tag !== "plist") throw new Error("Not a property list."); + if (root.empty) return {}; + const top = value(); + if (typeof top !== "object" || top === null || Array.isArray(top) || top instanceof PlistData || top instanceof PlistDate || typeof top === "bigint") throw new Error("The property list root is not a dictionary."); + return top; +} + +function plainPlist(value: PlistValue): unknown { + if (value instanceof PlistData) return `<${value.bytes.length} bytes>`; + if (value instanceof PlistDate) return value.iso; + if (typeof value === "bigint") return value.toString(); + if (Array.isArray(value)) return value.map(plainPlist); + if (typeof value === "object") return Object.fromEntries(Object.entries(value).map(([key, child]) => [key, plainPlist(child)])); + return value; +} + +export function plistEntries(dict: Record): ParsedEntry[] { + return Object.entries(dict).map(([key, value]) => { + if (value instanceof PlistData) return { key, type: "data", value: value.bytes.toString("hex"), bytes: value.bytes.length }; + if (value instanceof PlistDate) return { key, type: "date", value: value.iso, bytes: value.iso.length }; + if (typeof value === "boolean") return { key, type: "boolean", value: String(value), bytes: 1 }; + if (typeof value === "bigint") return { key, type: "integer", value: value.toString(), bytes: 8 }; + if (typeof value === "number") return { key, type: Number.isInteger(value) ? "integer" : "number", value: String(value), bytes: 8 }; + if (typeof value === "string") return { key, type: jsonType(value) ?? "string", value, bytes: Buffer.byteLength(value) }; + const json = JSON.stringify(plainPlist(value)); + return { key, type: Array.isArray(value) ? "array" : "object", value: json, bytes: Buffer.byteLength(json) }; + }); +} + +// ---- Android SharedPreferences XML. + +export function parseSharedPreferences(xml: string): ParsedEntry[] { + const entries: ParsedEntry[] = []; + const body = xml.replace(/<\?[\s\S]*?\?>|/g, ""); + const element = /<(string|boolean|int|long|float|null|set)\s+name="([^"]*)"(?:\s+value="([^"]*)")?\s*(?:\/>|>([\s\S]*?)<\/\1>)/g; + for (const match of body.matchAll(element)) { + const [, tag, rawName, rawValue, content = ""] = match; + const key = unescapeXml(rawName); + if (tag === "string") { + const value = unescapeXml(content); + entries.push({ key, type: jsonType(value) ?? "string", value, bytes: Buffer.byteLength(value) }); + } else if (tag === "set") { + const items = [...content.matchAll(/([\s\S]*?)<\/string>/g)].map(item => unescapeXml(item[1])); + const value = JSON.stringify(items); + entries.push({ key, type: "array", value, bytes: Buffer.byteLength(value) }); + } else if (tag === "null") entries.push({ key, type: "null", value: "null", bytes: 0 }); + else { + const value = unescapeXml(rawValue ?? ""); + const type: EntryType = tag === "boolean" ? "boolean" : tag === "float" ? "number" : "integer"; + entries.push({ key, type, value, bytes: tag === "long" ? 8 : tag === "boolean" ? 1 : 4 }); + } + } + return entries; +} + +// ---- Jetpack DataStore Preferences: protobuf PreferenceMap { map preferences = 1; }. + +type Field = { number: number; wire: number; varint?: bigint; bytes?: Buffer }; +function fields(buffer: Buffer): Field[] { + const result: Field[] = []; + let offset = 0; + while (offset < buffer.length) { + const tag = readVarint(buffer, offset); + if (tag === undefined) throw new Error("Truncated protobuf tag."); + offset = tag.next; + const number = Number(tag.value >> 3n); + const wire = Number(tag.value & 7n); + if (wire === 0) { + const value = readVarint(buffer, offset); + if (value === undefined) throw new Error("Truncated protobuf varint."); + result.push({ number, wire, varint: value.value }); + offset = value.next; + } else if (wire === 2) { + const length = readVarint(buffer, offset); + if (length === undefined) throw new Error("Truncated protobuf length."); + const end = length.next + Number(length.value); + if (end > buffer.length) throw new Error("Protobuf field runs past the end."); + result.push({ number, wire, bytes: buffer.subarray(length.next, end) }); + offset = end; + } else if (wire === 5 || wire === 1) { + const size = wire === 5 ? 4 : 8; + if (offset + size > buffer.length) throw new Error("Truncated protobuf fixed field."); + result.push({ number, wire, bytes: buffer.subarray(offset, offset + size) }); + offset += size; + } else throw new Error(`Unsupported protobuf wire type ${wire}.`); + } + return result; +} + +export function parseDataStorePreferences(buffer: Buffer): ParsedEntry[] { + return fields(buffer).filter(field => field.number === 1 && field.bytes).flatMap(field => { + const pair = fields(field.bytes!); + const key = pair.find(item => item.number === 1)?.bytes?.toString("utf8"); + const value = pair.find(item => item.number === 2)?.bytes; + if (key === undefined || value === undefined) return []; + const [inner] = fields(value); + if (inner === undefined) return [{ key, type: "null", value: "null", bytes: 0 } satisfies ParsedEntry]; + const entry = (type: EntryType, text: string, bytes: number): ParsedEntry => ({ key, type, value: text, bytes }); + switch (inner.number) { + case 1: return [entry("boolean", String(inner.varint === 1n), 1)]; + case 2: return [entry("number", String(inner.bytes!.readFloatLE(0)), 4)]; + case 3: return [entry("integer", BigInt.asIntN(32, inner.varint!).toString(), 4)]; + case 4: return [entry("integer", BigInt.asIntN(64, inner.varint!).toString(), 8)]; + case 5: { const text = inner.bytes!.toString("utf8"); return [entry(jsonType(text) ?? "string", text, inner.bytes!.length)]; } + case 6: { const json = JSON.stringify(fields(inner.bytes!).map(item => item.bytes?.toString("utf8") ?? "")); return [entry("array", json, Buffer.byteLength(json))]; } + case 7: return [entry("number", String(inner.bytes!.readDoubleLE(0)), 8)]; + case 8: return [entry("data", inner.bytes!.toString("hex"), inner.bytes!.length)]; + default: return []; + } + }); +} + +// ---- Classifying files in the data directory. + +export type StoreCandidate = { kind: StoreKind; name: string; path: string; files: string[] }; + +// System caches the app does not write itself; they stay in the file list but are not shown as databases. +const systemPaths = /^(?:Library\/(?:Caches|HTTPStorages|WebKit|Cookies|SplashBoard|Saved Application State)\/|SystemData\/|cache\/|code_cache\/|app_webview\/|no_backup\/androidx\.work)/; +const sqliteSuffix = /-(?:wal|shm|journal)$/; + +const basename = (path: string) => path.slice(path.lastIndexOf("/") + 1); +const dirname = (path: string) => path.includes("/") ? path.slice(0, path.lastIndexOf("/")) : ""; + +export function classifyFiles(paths: readonly string[], platform: "ios" | "android"): StoreCandidate[] { + const all = new Set(paths); + const stores: StoreCandidate[] = []; + for (const path of paths) { + const name = basename(path); + const directory = dirname(path); + // An MMKV instance is a data file with a `.crc` file next to it, wherever the app placed it. + if (name.endsWith(".crc") && all.has(path.slice(0, -4))) { + stores.push({ kind: "mmkv", name: name.slice(0, -4), path: path.slice(0, -4), files: [path.slice(0, -4), path] }); + continue; + } + if (platform === "ios" && directory === "Library/Preferences" && name.endsWith(".plist")) { + stores.push({ kind: "user-defaults", name: name.slice(0, -6), path, files: [path] }); + continue; + } + if (name === "manifest.json" && /^RCTAsyncLocalStorage/.test(basename(directory))) { + stores.push({ kind: "async-storage", name: directory.includes("ExponentExperienceData/") ? directory.split("/").at(-2) ?? "default" : "default", path, files: paths.filter(file => dirname(file) === directory) }); + continue; + } + if (directory.endsWith("shared_prefs") && name.endsWith(".xml")) { + stores.push({ kind: "shared-preferences", name: name.slice(0, -4), path, files: [path, ...all.has(`${path}.bak`) ? [`${path}.bak`] : []] }); + continue; + } + if (name.endsWith(".preferences_pb")) { + stores.push({ kind: "datastore", name: name.slice(0, -".preferences_pb".length), path, files: [path] }); + continue; + } + if (name.endsWith(".realm")) { + const related = paths.filter(file => file !== path && (file.startsWith(`${path}.`) || file.startsWith(`${path}.management/`))); + stores.push({ kind: "realm", name: name.slice(0, -6), path, files: [path, ...related] }); + continue; + } + const sqliteName = platform === "android" ? directory === "databases" && !sqliteSuffix.test(name) && !name.endsWith(".bak") : /\.(?:db|sqlite3?|sql)$/i.test(name); + if ((sqliteName || /\.(?:db|sqlite3?)$/i.test(name)) && !systemPaths.test(path)) { + stores.push({ kind: "sqlite", name, path, files: [path, ...["-wal", "-shm", "-journal"].map(suffix => path + suffix).filter(file => all.has(file))] }); + } + } + return stores; +} + +export function storeId(candidate: Pick) { return `${candidate.kind}:${candidate.path}`; } diff --git a/src/server/app-storage/mmkv.ts b/src/server/app-storage/mmkv.ts new file mode 100644 index 00000000..bf229e62 --- /dev/null +++ b/src/server/app-storage/mmkv.ts @@ -0,0 +1,121 @@ +import { crc32 } from "node:zlib"; +import type { EntryType } from "../../shared/app-storage.ts"; + +// MMKV's on-disk format, from Tencent/MMKV Core (MMKV_IO.cpp, MiniPBCoder.cpp, MMKVMetaInfo.hpp): +// - `` holds a 4-byte header, then a placeholder varint and appended records of +// varint key length, key, varint value length, value. Later records win; an empty value removes the key. +// - `.crc` holds the CRC, version, write sequence, AES IV, and since version 3 the data size. +// Version 4 adds flags at byte 104; flag 1 appends a 4-byte expiry time to every value. +// Values carry no type, so callers guess one from the bytes. + +export type MmkvRecord = { key: string; value: Buffer }; +export type MmkvFile = + | { status: "ok"; records: MmkvRecord[]; crcValid: boolean | undefined } + | { status: "encrypted" } + | { status: "unreadable"; reason: string }; + +const versionRandomIv = 2; +const versionActualSize = 3; +const versionFlag = 4; +const flagKeyExpire = 1n; + +export function readVarint(buffer: Buffer, offset: number, limit = 10): { value: bigint; next: number } | undefined { + let value = 0n; + let shift = 0n; + for (let index = 0; index < limit; index++) { + if (offset + index >= buffer.length) return undefined; + const byte = buffer[offset + index]; + value |= BigInt(byte & 0x7f) << shift; + if ((byte & 0x80) === 0) return { value, next: offset + index + 1 }; + shift += 7n; + } + return undefined; +} + +function lengthDelimited(buffer: Buffer, offset: number) { + const length = readVarint(buffer, offset, 5); + if (length === undefined) throw new Error("truncated length"); + const end = length.next + Number(length.value); + if (end > buffer.length) throw new Error("record runs past the data size"); + return { bytes: buffer.subarray(length.next, end), next: end }; +} + +export function decodeMmkv(data: Buffer, meta?: Buffer): MmkvFile { + const version = meta && meta.length >= 8 ? meta.readUInt32LE(4) : 0; + if (meta && meta.length >= 28 && version >= versionRandomIv && meta.subarray(12, 28).some(byte => byte !== 0)) return { status: "encrypted" }; + if (data.length < 4) return { status: "ok", records: [], crcValid: undefined }; + const size = meta && meta.length >= 32 && version >= versionActualSize ? meta.readUInt32LE(28) : data.readUInt32LE(0); + if (size + 4 > data.length) return { status: "unreadable", reason: "The file is shorter than its recorded size." }; + const body = data.subarray(4, 4 + size); + const crcValid = meta && meta.length >= 4 ? crc32(body) === meta.readUInt32LE(0) : undefined; + const expiring = meta !== undefined && meta.length >= 112 && version >= versionFlag && (meta.readBigUInt64LE(104) & flagKeyExpire) !== 0n; + const now = Math.floor(Date.now() / 1000); + const records = new Map(); + try { + let offset = 0; + if (body.length > 0) { + const placeholder = readVarint(body, 0, 5); + if (placeholder === undefined) throw new Error("missing header"); + offset = placeholder.next; + } + while (offset < body.length) { + const key = lengthDelimited(body, offset); + offset = key.next; + // MMKV skips records with an empty key without reading a value. + if (key.bytes.length === 0) continue; + const value = lengthDelimited(body, offset); + offset = value.next; + const name = key.bytes.toString("utf8"); + let bytes = value.bytes; + if (expiring && bytes.length >= 4) { + const expiresAt = bytes.readUInt32LE(bytes.length - 4); + bytes = bytes.subarray(0, bytes.length - 4); + if (expiresAt !== 0 && expiresAt <= now) { records.delete(name); continue; } + } + if (value.bytes.length === 0) records.delete(name); + else records.set(name, bytes); + } + } catch (error) { + return { status: "unreadable", reason: `The data does not decode as MMKV (${error instanceof Error ? error.message : "invalid"}).` }; + } + return { status: "ok", records: [...records].map(([key, value]) => ({ key, value })), crcValid }; +} + +const utf8 = new TextDecoder("utf-8", { fatal: true }); +function text(bytes: Buffer): string | undefined { + try { + const value = utf8.decode(bytes); + // Control characters other than whitespace mean binary data that happens to be valid UTF-8. + return /[\u0000-\u0008\u000b\u000c\u000e-\u001f]/.test(value) ? undefined : value; + } catch { return undefined; } +} + +export function jsonType(value: string): "json" | undefined { + const trimmed = value.trim(); + if (!/^[[{]/.test(trimmed)) return undefined; + try { JSON.parse(trimmed); return "json"; } catch { return undefined; } +} + +// Same tie-breaking order as Rozenite's MMKV adapter: strings, then byte buffers, then numbers, then booleans. +// A single 0x00 byte is either false or an empty string; it reads as false. +export function guessMmkvValue(bytes: Buffer): { type: EntryType; value: string } { + const prefix = readVarint(bytes, 0, 5); + if (bytes.length > 1 && prefix && prefix.next + Number(prefix.value) === bytes.length) { + const payload = bytes.subarray(prefix.next); + const string = text(payload); + if (string !== undefined) return { type: jsonType(string) ?? "string", value: string }; + return { type: "data", value: payload.toString("hex") }; + } + if (bytes.length === 1 && bytes[0] <= 1) return { type: "boolean", value: bytes[0] === 1 ? "true" : "false" }; + if (bytes.length === 8) { + const number = bytes.readDoubleLE(0); + if (Number.isFinite(number)) return { type: "number", value: String(number) }; + } + const varint = readVarint(bytes, 0); + if (varint && varint.next === bytes.length) return { type: "integer", value: BigInt.asIntN(64, varint.value).toString() }; + if (bytes.length === 4) { + const number = bytes.readFloatLE(0); + if (Number.isFinite(number)) return { type: "number", value: String(number) }; + } + return { type: "data", value: bytes.toString("hex") }; +} diff --git a/src/server/app-storage/reader.ts b/src/server/app-storage/reader.ts new file mode 100644 index 00000000..02183802 --- /dev/null +++ b/src/server/app-storage/reader.ts @@ -0,0 +1,158 @@ +import { createHash } from "node:crypto"; +import { join } from "node:path"; +import { ExpectedOperationError } from "../../shared/error-reporting.ts"; +import { storageLimits } from "../../shared/app-storage.ts"; +import type { StorageEntry, StorageFile, StorageSnapshot, StorageStore } from "../../shared/app-storage.ts"; +import { withTemporaryDirectory } from "../app-control/commands.ts"; +import type { CommandRunner } from "../app-control/commands.ts"; +import type { AppContainer, ContainerFile } from "./container.ts"; +import { classifyFiles, parseDataStorePreferences, parsePlistXml, parseSharedPreferences, plistEntries, storeId } from "./formats.ts"; +import type { ParsedEntry, StoreCandidate } from "./formats.ts"; +import { decodeMmkv, guessMmkvValue, jsonType } from "./mmkv.ts"; + +export type ReadOptions = { entries: number; valueLength: number; bundleId: string; run: CommandRunner }; +type StoreContent = { status: StorageStore["status"]; note?: string; entries: ParsedEntry[] }; + +const sqlite3 = process.platform === "darwin" ? "/usr/bin/sqlite3" : "sqlite3"; +const quoteIdentifier = (name: string) => `"${name.replaceAll("\"", "\"\"")}"`; + +async function sqlite(run: CommandRunner, file: string, sql: string): Promise[]> { + const output = await run(sqlite3, ["-json", file, sql], { timeout: 15000, maxBuffer: 64 * 1024 * 1024 }); + return output.trim() ? JSON.parse(output) : []; +} + +function keyValueEntries(rows: Record[]): ParsedEntry[] { + return rows.map(row => { + const value = row.value === null || row.value === undefined ? null : String(row.value); + if (value === null) return { key: String(row.key), type: "null", value: "null", bytes: 0 }; + return { key: String(row.key), type: jsonType(value) ?? "string", value, bytes: Buffer.byteLength(value) }; + }); +} + +// AsyncStorage tables are recognized by name: v1/v2 use catalystLocalStorage, v2's next storage a Room +// table named Storage, and v3 an `entries` table under async-storage/databases/. +async function readSqlite(container: AppContainer, candidate: StoreCandidate, run: CommandRunner): Promise { + return withTemporaryDirectory(async directory => { + // The copy includes the write-ahead log, so recent writes that SQLite has not checkpointed are read too. + await container.copy(candidate.files, directory); + const file = join(directory, candidate.name); + let tables: string[]; + try { tables = (await sqlite(run, file, "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name")).map(row => String(row.name)); } + catch { return { status: "unreadable", note: "Not a readable SQLite database.", entries: [] }; } + const keyValueTable = tables.includes("catalystLocalStorage") ? "catalystLocalStorage" + : tables.includes("entries") && candidate.path.includes("async-storage/") ? "entries" + : tables.includes("Storage") && /^AsyncStorage/.test(candidate.name) ? "Storage" : undefined; + if (keyValueTable) { + const rows = await sqlite(run, file, `SELECT key, value FROM ${quoteIdentifier(keyValueTable)}`); + return { status: "ok", entries: keyValueEntries(rows), asyncStorage: true }; + } + const entries = await Promise.all(tables.map(async (table): Promise => { + const [count] = await sqlite(run, file, `SELECT count(*) AS rows FROM ${quoteIdentifier(table)}`).catch(() => [{ rows: "?" }]); + const columns = (await sqlite(run, file, `PRAGMA table_info(${quoteIdentifier(table)})`).catch(() => [])).map(column => String(column.name)); + const value = `${count?.rows ?? "?"} rows · ${columns.join(", ")}`; + return { key: table, type: "table", value, bytes: 0 }; + })); + return { status: "ok", entries }; + }); +} + +async function readAsyncManifest(container: AppContainer, candidate: StoreCandidate): Promise { + const manifest: unknown = JSON.parse((await container.read(candidate.path)).toString("utf8")); + if (typeof manifest !== "object" || manifest === null) return { status: "unreadable", note: "The AsyncStorage manifest is not an object.", entries: [] }; + const directory = candidate.path.slice(0, candidate.path.lastIndexOf("/")); + // Values over 1 KB live in a file named after the MD5 of their key; the manifest stores null for them. + const entries = await Promise.all(Object.entries(manifest).map(async ([key, stored]): Promise => { + const value = typeof stored === "string" ? stored + : await container.read(`${directory}/${createHash("md5").update(key).digest("hex")}`).then(buffer => buffer.toString("utf8"), () => null); + if (value === null) return { key, type: "null", value: "null", bytes: 0 }; + return { key, type: jsonType(value) ?? "string", value, bytes: Buffer.byteLength(value) }; + })); + return { status: "ok", entries }; +} + +async function readStore(container: AppContainer, candidate: StoreCandidate, run: CommandRunner): Promise { + switch (candidate.kind) { + case "mmkv": { + const [data, meta] = await Promise.all(candidate.files.map(file => container.read(file))); + let decoded = decodeMmkv(data, meta); + // The app may be writing while the files are read; a CRC mismatch is retried once. + if (decoded.status === "ok" && decoded.crcValid === false) { + const [again, againMeta] = await Promise.all(candidate.files.map(file => container.read(file))); + decoded = decodeMmkv(again, againMeta); + } + if (decoded.status === "encrypted") return { status: "encrypted", note: "Encrypted with a key only the app knows.", entries: [] }; + if (decoded.status === "unreadable") return { status: "unreadable", note: decoded.reason, entries: [] }; + const note = decoded.crcValid === false ? "The checksum does not match; the app may be writing. Refresh to read again." : undefined; + return { status: "ok", note, entries: decoded.records.map(record => ({ key: record.key, ...guessMmkvValue(record.value), bytes: record.value.length, guessed: true })) }; + } + case "user-defaults": { + const xml = container.preferences ? await container.preferences(candidate.path) : (await container.read(candidate.path)).toString("utf8"); + return { status: "ok", entries: plistEntries(parsePlistXml(xml)) }; + } + case "shared-preferences": + return { status: "ok", entries: parseSharedPreferences((await container.read(candidate.path)).toString("utf8")) }; + case "datastore": + return { status: "ok", entries: parseDataStorePreferences(await container.read(candidate.path)) }; + case "async-storage": + return readAsyncManifest(container, candidate); + case "sqlite": + return readSqlite(container, candidate, run); + case "realm": + return { status: "ok", note: "Realm files are listed but not decoded.", entries: [] }; + } +} + +function limitEntry(entry: ParsedEntry, valueLength: number): StorageEntry { + if (entry.value.length <= valueLength) return entry; + return { ...entry, value: entry.value.slice(0, valueLength), truncated: true }; +} + +async function mapLimited(items: readonly T[], limit: number, work: (item: T) => Promise): Promise { + const results: R[] = new Array(items.length); + let next = 0; + await Promise.all(Array.from({ length: Math.min(limit, items.length) }, async () => { + while (next < items.length) { const index = next++; results[index] = await work(items[index]); } + })); + return results; +} + +export async function readStorage(container: AppContainer, { entries: entryLimit, valueLength, bundleId, run }: ReadOptions): Promise { + const files = await container.list(); + const sizes = new Map(files.map(file => [file.path, file.bytes])); + const candidates = classifyFiles(files.map(file => file.path), container.platform); + const owner = new Map(); + const stores = await mapLimited(candidates, 4, async (candidate): Promise => { + const id = storeId(candidate); + for (const file of candidate.files) owner.set(file, id); + let content: StoreContent & { asyncStorage?: boolean }; + try { content = await readStore(container, candidate, run); } + catch (error) { + if (error instanceof ExpectedOperationError) throw error; + content = { status: "unreadable", note: "This file could not be read or parsed.", entries: [] }; + } + const kind = content.asyncStorage ? "async-storage" : candidate.kind; + return { + id, kind, name: candidate.name, path: candidate.path, status: content.status, ...content.note ? { note: content.note } : {}, + bytes: candidate.files.reduce((total, file) => total + (sizes.get(file) ?? 0), 0), + entryCount: content.entries.length, + entries: content.entries.slice(0, entryLimit).map(entry => limitEntry(entry, valueLength)), + }; + }); + const sorted = [...files].sort((a, b) => b.bytes - a.bytes).slice(0, storageLimits.files); + return { + platform: container.platform, bundleId, readAt: Date.now(), stores, + files: sorted.map((file: ContainerFile): StorageFile => ({ ...file, ...owner.has(file.path) ? { store: owner.get(file.path) } : {} })), + fileCount: files.length, + totalBytes: files.reduce((total, file) => total + file.bytes, 0), + }; +} + +// Store IDs are matched against a fresh listing, so a caller can only remove files that belong to a found store. +export async function clearStore(container: AppContainer, id: string) { + const files = await container.list(); + const candidate = classifyFiles(files.map(file => file.path), container.platform).find(store => storeId(store) === id); + if (candidate === undefined) throw new ExpectedOperationError("invalid_input", "That storage no longer exists. Refresh and try again."); + if (candidate.kind === "user-defaults" && container.forgetPreferences) await container.forgetPreferences(candidate.path); + await container.remove(candidate.files); + return candidate; +} diff --git a/src/server/app-storage/tools.ts b/src/server/app-storage/tools.ts new file mode 100644 index 00000000..3c44acba --- /dev/null +++ b/src/server/app-storage/tools.ts @@ -0,0 +1,85 @@ +import { z } from "zod"; +import { registerAppTool } from "@modelcontextprotocol/ext-apps/server"; +import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import { captureServerError, recordAppStorageOperation } from "../telemetry.ts"; +import { errorMessage } from "../../shared/protocol.ts"; +import { expectedOutcome } from "../../shared/error-reporting.ts"; +import { appTargetShape, bundleIdSchema } from "../../shared/app-control.ts"; +import { storageLimits } from "../../shared/app-storage.ts"; +import { systemCommands } from "../app-control/commands.ts"; +import type { Commands } from "../app-control/commands.ts"; +import { createDrivers } from "../app-control/tools.ts"; +import type { DriverFactory } from "../app-control/tools.ts"; +import { AndroidContainer, SimulatorContainer } from "./container.ts"; +import type { AppContainer } from "./container.ts"; +import { clearStore, readStorage } from "./reader.ts"; + +type Target = { platform: "ios" | "android"; deviceId: string }; +export type ContainerFactory = (target: Target, bundleId: string) => Promise; + +export function createContainers(commands: Commands = systemCommands): ContainerFactory { + return async ({ platform, deviceId }, bundleId) => platform === "android" + ? new AndroidContainer(deviceId, bundleId, commands) + : SimulatorContainer.open(deviceId, bundleId, commands); +} + +const bundleId = bundleIdSchema.describe("App bundle ID or Android package name."); + +export function registerAppStorageTools(server: McpServer, commands: Commands = systemCommands, driverFor: DriverFactory = createDrivers(commands), containerFor: ContainerFactory = createContainers(commands)) { + async function run(operation: "read" | "clear" | "reset", target: Target, work: () => Promise>) { + const startedAt = performance.now(); + try { + const result = await work(); + recordAppStorageOperation(operation, target.platform, "success", performance.now() - startedAt); + return { content: [{ type: "text" as const, text: JSON.stringify(result) }], structuredContent: result }; + } catch (error) { + recordAppStorageOperation(operation, target.platform, "failed", performance.now() - startedAt); + // Paths, keys, and values can be personal, so Sentry receives only the operation. + captureServerError(expectedOutcome(error) === undefined ? new Error(`App storage ${operation} failed.`) : error, `app_storage.${operation}`); + return { isError: true, content: [{ type: "text" as const, text: errorMessage(error) }] }; + } + } + + registerAppTool(server, "mobile_app_storage", { + title: "Read app storage", + description: "Read an app's saved data without changes to the app: MMKV, AsyncStorage, UserDefaults, SharedPreferences, DataStore, SQLite databases, and the largest files. iOS simulators, and debuggable Android builds. MMKV value types are guessed from bytes.", + inputSchema: { + ...appTargetShape, bundleId, + store: z.string().max(4096).optional().describe("Only return this store ID from an earlier result."), + entries: z.number().int().min(1).max(storageLimits.maxEntries).optional().describe(`Entries per store, default ${storageLimits.entries}.`), + valueLength: z.number().int().min(40).max(storageLimits.maxValueLength).optional().describe(`Characters per value, default ${storageLimits.valueLength}.`), + }, + annotations: { readOnlyHint: true, destructiveHint: false, openWorldHint: false }, + _meta: { ui: { visibility: ["app", "model"] } }, + }, async input => { + const target = { platform: input.platform, deviceId: input.deviceId }; + return run("read", target, async () => { + driverFor(target); + const container = await containerFor(target, input.bundleId); + const snapshot = await readStorage(container, { bundleId: input.bundleId, entries: input.entries ?? storageLimits.entries, valueLength: input.valueLength ?? storageLimits.valueLength, run: commands.run }); + return input.store ? { ...snapshot, stores: snapshot.stores.filter(store => store.id === input.store) } : snapshot; + }); + }); + + registerAppTool(server, "mobile_clear_app_storage", { + title: "Clear app storage", + description: "Stop the app, then delete one store from mobile_app_storage, or all of the app's saved data when store is omitted. The app stays installed and is not relaunched.", + inputSchema: { ...appTargetShape, bundleId, store: z.string().max(4096).optional().describe("Store ID to clear. Omit to reset all app data.") }, + annotations: { readOnlyHint: false, destructiveHint: true, openWorldHint: false }, + _meta: { ui: { visibility: ["app", "model"] } }, + }, async input => { + const target = { platform: input.platform, deviceId: input.deviceId }; + return run(input.store ? "clear" : "reset", target, async () => { + const driver = driverFor(target); + const container = await containerFor(target, input.bundleId); + // A running app keeps values in memory and would write them back. + await driver.stop(input.bundleId); + if (input.store === undefined) { + await container.reset(); + return { cleared: "all", stopped: true }; + } + const store = await clearStore(container, input.store); + return { cleared: store.name, stopped: true }; + }); + }); +} diff --git a/src/server/plugin.ts b/src/server/plugin.ts index ff671ffb..d592bdc8 100644 --- a/src/server/plugin.ts +++ b/src/server/plugin.ts @@ -16,6 +16,7 @@ import { OpenAIExtensions } from "@openai/mcp-extensions/server"; import { z } from "zod"; import { registerDeviceSettingsTools } from "./device-settings.ts"; import { registerAppControlTools } from "./app-control/tools.ts"; +import { registerAppStorageTools } from "./app-storage/tools.ts"; import { readBezel } from "./bezel.ts"; import { ServeEmu } from "./serve-emu.ts"; import { registerAndroidTools } from "./android-tools.ts"; @@ -123,6 +124,7 @@ export async function createPlugin(html: string | (() => Promise), b }); const closeDeviceSettings = registerDeviceSettingsTools(server, baguette, android); registerAppControlTools(server); + registerAppStorageTools(server); const automaticRepairs = new Map }>(); async function panelInputStatus(udid: string) { diff --git a/src/server/telemetry.ts b/src/server/telemetry.ts index 9c5fa2ce..0c98d2bc 100644 --- a/src/server/telemetry.ts +++ b/src/server/telemetry.ts @@ -27,6 +27,13 @@ export function recordPluginUpdate(operation: "check" | "install", outcome: "dis Sentry.metrics.gauge("plugin.update.duration", duration, { unit: "millisecond", attributes }); } +export function recordAppStorageOperation(operation: "read" | "clear" | "reset", platform: "ios" | "android", outcome: "success" | "failed", duration: number) { + if (process.env.MOBILE_DEV_TELEMETRY === "off") return; + const attributes = { component: "server", surface: "storage", operation, device_platform: platform, outcome }; + Sentry.metrics.count("app_storage.operation.samples", 1, { attributes }); + Sentry.metrics.gauge("app_storage.operation.duration", duration, { unit: "millisecond", attributes }); +} + export function recordTraceSiftOperation(operation: "ready" | "analyze" | "remember" | "restore" | "handoff" | "task", outcome: "success" | "failed", duration: number) { if (process.env.MOBILE_DEV_TELEMETRY === "off") return; const attributes = { component: "server", surface: "tracesift", operation, outcome }; @@ -233,7 +240,7 @@ const resourceKinds: Record = { "android-stream:": "android_video", "ios-video:": "ios_video", "ui:": "ui", }; const allowedContext: Record = { - surface: ["logs", "performance", "simulator", "recording", "comparison"], view: ["panel", "workspace", "recording", "comparison"], layout: ["ios", "android", "both", "none"], + surface: ["logs", "performance", "storage", "simulator", "recording", "comparison"], view: ["panel", "workspace", "recording", "comparison"], layout: ["ios", "android", "both", "none"], device_platform: ["ios", "android", "mixed"], device_kind: ["physical", "simulator", "emulator", "none"], }; diff --git a/src/shared/app-storage.ts b/src/shared/app-storage.ts new file mode 100644 index 00000000..3f79ec78 --- /dev/null +++ b/src/shared/app-storage.ts @@ -0,0 +1,59 @@ +import { z } from "zod"; + +// Storage the panel recognizes from files in the app's data directory, without code in the app. +export const storeKinds = ["mmkv", "async-storage", "user-defaults", "shared-preferences", "datastore", "sqlite", "realm"] as const; +export type StoreKind = typeof storeKinds[number]; +export const storeKindLabels: Record = { + "mmkv": "MMKV", "async-storage": "AsyncStorage", "user-defaults": "UserDefaults", "shared-preferences": "SharedPreferences", + "datastore": "DataStore", "sqlite": "Databases", "realm": "Realm", +}; + +export const entryTypes = ["string", "json", "number", "integer", "boolean", "date", "data", "array", "object", "null", "table"] as const; +export type EntryType = typeof entryTypes[number]; + +// Values arrive as display text: JSON for json, arrays and objects, hex for data. `guessed` marks a type +// inferred from bytes, because MMKV files do not record value types. +const entry = z.object({ + key: z.string(), + type: z.enum(entryTypes), + value: z.string(), + bytes: z.number().int().nonnegative(), + guessed: z.boolean().optional(), + truncated: z.boolean().optional(), +}); +export type StorageEntry = z.infer; + +// `encrypted` stores are found but cannot be read without the app's key; `unreadable` ones failed to parse. +export const storeStatuses = ["ok", "encrypted", "unreadable"] as const; +const store = z.object({ + id: z.string(), + kind: z.enum(storeKinds), + name: z.string(), + path: z.string(), + status: z.enum(storeStatuses), + note: z.string().optional(), + bytes: z.number().int().nonnegative(), + entryCount: z.number().int().nonnegative(), + entries: z.array(entry), +}); +export type StorageStore = z.infer; + +const file = z.object({ path: z.string(), bytes: z.number().int().nonnegative(), store: z.string().optional() }); +export type StorageFile = z.infer; + +export const storageSnapshotSchema = z.object({ + platform: z.enum(["ios", "android"]), + bundleId: z.string(), + readAt: z.number(), + stores: z.array(store), + // Largest files first, capped; fileCount and totalBytes cover the whole data directory. + files: z.array(file), + fileCount: z.number().int().nonnegative(), + totalBytes: z.number().int().nonnegative(), +}); +export type StorageSnapshot = z.infer; + +export const storageClearSchema = z.object({ cleared: z.string(), stopped: z.boolean() }); + +// Defaults keep model-visible results small; the panel asks for more. +export const storageLimits = { entries: 200, maxEntries: 2000, valueLength: 300, maxValueLength: 16000, files: 400 }; diff --git a/src/shared/telemetry.ts b/src/shared/telemetry.ts index 342c0b27..1ded4b90 100644 --- a/src/shared/telemetry.ts +++ b/src/shared/telemetry.ts @@ -17,7 +17,7 @@ export function validateTelemetryEnvironment(value: unknown): TelemetryEnvironme throw new Error("Sentry environment must be development or release."); } -export type Surface = "logs" | "performance" | "simulator" | "recording" | "comparison" | "tracesift"; +export type Surface = "logs" | "performance" | "storage" | "simulator" | "recording" | "comparison" | "tracesift"; export type TelemetryAttributes = Record; type Options = NonNullable[0]>; type ErrorEvent = Parameters>[0]; diff --git a/src/ui/components/storage-view.tsx b/src/ui/components/storage-view.tsx new file mode 100644 index 00000000..6a8fc3df --- /dev/null +++ b/src/ui/components/storage-view.tsx @@ -0,0 +1,237 @@ +import { CopyIcon, DatabaseIcon, FolderIcon, LockIcon, PlayIcon, RefreshCwIcon, RotateCcwIcon, SearchIcon, TriangleAlertIcon } from "lucide-react"; +import { memo, useEffect, useMemo, useState, useSyncExternalStore } from "react"; +import { storeKindLabels, storeKinds } from "../../shared/app-storage.ts"; +import type { StorageEntry, StorageStore } from "../../shared/app-storage.ts"; +import type { StoragePanel } from "../storage-panel.ts"; +import { Alert, AlertDescription } from "./ui/alert"; +import { Badge } from "./ui/badge"; +import { Button } from "./ui/button"; +import { Empty, EmptyDescription, EmptyHeader, EmptyMedia, EmptyTitle } from "./ui/empty"; +import { InputGroup, InputGroupAddon, InputGroupInput } from "./ui/input-group"; +import { NativeSelect, NativeSelectOption } from "./ui/native-select"; +import { Popover, PopoverClose, PopoverContent, PopoverTrigger } from "./ui/popover"; +import { ResizableHandle, ResizablePanel, ResizablePanelGroup } from "./ui/resizable"; +import { Toggle } from "./ui/toggle"; +import { useMediaQuery } from "./use-media-query"; + +const FILES = "files"; + +export function formatBytes(bytes: number) { + if (bytes < 1024) return `${bytes} B`; + if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`; + return `${(bytes / 1024 / 1024).toFixed(1)} MB`; +} + +function formatAge(milliseconds: number) { + const seconds = Math.max(0, Math.round(milliseconds / 1000)); + if (seconds < 60) return `${seconds}s ago`; + const minutes = Math.round(seconds / 60); + return minutes < 60 ? `${minutes}m ago` : `${Math.round(minutes / 60)}h ago`; +} + +export function displayValue(entry: StorageEntry) { + if (entry.type === "data") return entry.value.replace(/(..)(?!$)/g, "$1 "); + if ((entry.type === "json" || entry.type === "array" || entry.type === "object") && !entry.truncated) { + try { return JSON.stringify(JSON.parse(entry.value), null, 2); } catch { return entry.value; } + } + return entry.value; +} + +const typeLabel = (entry: StorageEntry) => `${entry.type === "data" ? "bytes" : entry.type}${entry.guessed ? "?" : ""}`; + +function useNow(active: boolean) { + const [now, setNow] = useState(() => Date.now()); + useEffect(() => { + if (!active) return; + setNow(Date.now()); + const timer = setInterval(() => { if (document.visibilityState !== "hidden") setNow(Date.now()); }, 5000); + return () => clearInterval(timer); + }, [active]); + return now; +} + +export const StorageView = memo(function StorageView({ panel }: { panel: StoragePanel }) { + const state = useSyncExternalStore(panel.subscribe, panel.getSnapshot); + const wide = useMediaQuery("(min-width: 600px)"); + const [query, setQuery] = useState(""); + const [changedOnly, setChangedOnly] = useState(false); + const [selectedStore, setSelectedStore] = useState(""); + const [selectedKey, setSelectedKey] = useState(); + const [copied, setCopied] = useState(""); + const now = useNow(state.open && state.snapshot !== undefined); + const snapshot = state.snapshot; + const busy = state.busy !== null && state.busy !== "apps"; + const search = query.trim().toLowerCase(); + + const matches = useMemo(() => { + const result = new Map(); + for (const store of snapshot?.stores ?? []) { + const changes = state.changes.get(store.id); + result.set(store.id, store.entries.filter(entry => (!changedOnly || changes?.has(entry.key)) + && (!search || entry.key.toLowerCase().includes(search) || entry.value.toLowerCase().includes(search)))); + } + return result; + }, [snapshot, state.changes, search, changedOnly]); + + const filtering = search !== "" || changedOnly; + const visibleStores = (snapshot?.stores ?? []).filter(store => !filtering || (matches.get(store.id)?.length ?? 0) > 0); + const store = snapshot?.stores.find(item => item.id === selectedStore); + const activeStore = selectedStore === FILES ? undefined : store ?? visibleStores[0]; + const showFiles = selectedStore === FILES || (activeStore === undefined && snapshot !== undefined); + const entries = activeStore ? matches.get(activeStore.id) ?? [] : []; + const entry = entries.find(item => item.key === selectedKey); + const changes = activeStore ? state.changes.get(activeStore.id) : undefined; + const changedCount = [...state.changes.values()].reduce((total, marks) => total + marks.size, 0); + + const copy = async (text: string) => { + try { await navigator.clipboard.writeText(text); setCopied("Copied"); } + catch { setCopied("Could not copy"); } + }; + + const tree = ; + + const detail = entry && activeStore &&
+
+ {entry.key} + {typeLabel(entry)} · {formatBytes(entry.bytes)} + +
+ 
{displayValue(entry)}
+
+ {entry.guessed && "Type guessed from the stored bytes; MMKV does not record types. "} + {entry.truncated && "Value cut short; copy the full value with the mobile_app_storage tool. "} + {activeStore.path} +
+
; + + const storeView = activeStore &&
+
+ {storeKindLabels[activeStore.kind]} › {activeStore.name} + {activeStore.entryCount} {activeStore.kind === "sqlite" ? "table" : "key"}{activeStore.entryCount === 1 ? "" : "s"} · {formatBytes(activeStore.bytes)} + void panel.clear(activeStore.id)} /> +
+ {activeStore.note &&

{activeStore.status !== "ok" && (activeStore.status === "encrypted" ? : )}{activeStore.note}

} +
+ {entries.map(item => { + const change = changes?.get(item.key); + return ; + })} + {entries.length === 0 && activeStore.status === "ok" &&

{filtering ? "No matching keys." : "Empty."}

} + {activeStore.entries.length < activeStore.entryCount && !filtering &&

Showing the first {activeStore.entries.length} of {activeStore.entryCount}.

} +
+ {detail} +
; + + const filesView = snapshot &&
+
+ Files + {snapshot.fileCount} files · {formatBytes(snapshot.totalBytes)} +
+
+ {snapshot.files.filter(file => !search || file.path.toLowerCase().includes(search)).map(file => { + const owner = file.store ? snapshot.stores.find(item => item.id === file.store) : undefined; + return
+ {file.path} + {owner && {storeKindLabels[owner.kind]}} + {formatBytes(file.bytes)} +
; + })} + {snapshot.files.length < snapshot.fileCount &&

Showing the {snapshot.files.length} largest files.

} +
+
; + + let body; + if (state.unsupported || !state.available) { + body = ; + } else if (!state.bundleId) { + body = ; + } else if (!snapshot) { + body = state.error ? null : ; + } else if (snapshot.stores.length === 0 && !showFiles) { + body = ; + } else { + body = + {tree} + + {showFiles ? filesView : storeView} + ; + } + + return
+
+ + panel.selectApp(event.target.value)}> + {state.busy === "apps" ? "Loading apps…" : "Choose an app"} + {state.apps.map(app => {app.name === app.bundleId ? app.bundleId : `${app.name} · ${app.bundleId}`})} + + setQuery(event.target.value)} /> + Changed{changedCount > 0 && {changedCount}} + {snapshot && {state.busy === "read" ? "Reading…" : formatAge(now - snapshot.readAt)}} + + void panel.clear()} /> +
+ {copied} + {state.error && {state.error}} + {state.status &&
+ {state.status} + +
} + {body} +
; +}); + +function StoreButton({ store, count, changed, selected, onSelect }: { store: StorageStore; count: number; changed: boolean; selected: boolean; onSelect: () => void }) { + return ; +} + +function ConfirmButton({ label, title, description, disabled, onConfirm }: { label: string; title: string; description: string; disabled?: boolean; onConfirm: () => void }) { + return + + +

{title}?

+

{description}

+
+ + +
+ + ; +} + +function StorageEmpty({ title, description }: { title: string; description: string }) { + return + + + {title} + {description} + + ; +} diff --git a/src/ui/components/workspace.tsx b/src/ui/components/workspace.tsx index 7b947980..a77e3e88 100644 --- a/src/ui/components/workspace.tsx +++ b/src/ui/components/workspace.tsx @@ -1,10 +1,12 @@ -import { ActivityIcon, TerminalIcon, PanelsTopLeftIcon } from "lucide-react"; +import { ActivityIcon, DatabaseIcon, TerminalIcon, PanelsTopLeftIcon } from "lucide-react"; import { useEffect, useRef, useState, useSyncExternalStore } from "react"; import type { LogsPanel } from "../logs-panel.ts"; import type { PerformancePanel } from "../performance-panel.ts"; +import type { StoragePanel } from "../storage-panel.ts"; import { PerformanceView } from "./performance-view"; import { SimulatorView } from "./simulator-view"; import { LogsView } from "./logs-view"; +import { StorageView } from "./storage-view"; import { Select, SelectContent, SelectGroup, SelectItem, SelectTrigger, SelectValue } from "./ui/base-select"; import { Button } from "./ui/button"; @@ -20,8 +22,12 @@ import type { PluginUpdateController } from "../plugin-updates.ts"; import type { DeviceLayout } from "../device-layout.ts"; export type { DeviceLayout } from "../device-layout.ts"; -export function Workspace({ logs, performance, recordingController, updates, automaticLayout, onLayout }: { logs: LogsPanel; performance: PerformancePanel; recordingController: RecordingController; updates?: PluginUpdateController; automaticLayout?: DeviceLayout; onLayout: (layout: DeviceLayout, manual?: boolean) => void }) { - const [tool, setTool] = useState<"logs" | "performance">("logs"); +type Tool = "logs" | "performance" | "storage"; +const toolLabels: Record = { logs: "Logs", performance: "Performance", storage: "Storage" }; +const toolIcons = { logs: TerminalIcon, performance: ActivityIcon, storage: DatabaseIcon }; + +export function Workspace({ logs, performance, storage, recordingController, updates, automaticLayout, onLayout }: { logs: LogsPanel; performance: PerformancePanel; storage: StoragePanel; recordingController: RecordingController; updates?: PluginUpdateController; automaticLayout?: DeviceLayout; onLayout: (layout: DeviceLayout, manual?: boolean) => void }) { + const [tool, setTool] = useState("logs"); const [savedVisible, setSavedVisible] = useState(false); const [layout, setLayout] = useState(automaticLayout ?? "none"); const manualLayout = useRef(false); @@ -40,6 +46,7 @@ export function Workspace({ logs, performance, recordingController, updates, aut onLayout("none"); logs.hide(); performance.hide(); + storage.hide(); }, [recordingState.recording?.id]); const performanceState = useSyncExternalStore(performance.subscribe, performance.getSnapshot); const wide = useMediaQuery("(min-width: 900px)"); @@ -47,26 +54,30 @@ export function Workspace({ logs, performance, recordingController, updates, aut const split = wide && fullscreen; const collapsedToolSize = fullscreen ? (split ? 48 : 44) : 0; const logState = useSyncExternalStore(logs.subscribe, logs.getSnapshot); - const open = tool === "logs" ? logState.open : savedVisible || performanceState.open; + const storageState = useSyncExternalStore(storage.subscribe, storage.getSnapshot); + const open = tool === "logs" ? logState.open : tool === "storage" ? storageState.open : savedVisible || performanceState.open; useEffect(() => { setUiSurface(open ? savedVisible ? "recording" : tool : "simulator"); }, [tool, open, savedVisible]); useEffect(() => { setUiTelemetryContext({ logs_open: logState.open, logs_paused: logState.paused, performance_running: performanceState.monitoring || performanceState.fpsMonitoring }); }, [logState.open, logState.paused, performanceState.monitoring, performanceState.fpsMonitoring]); - const isOpen = () => tool === "logs" ? logs.getSnapshot().open : savedVisible || performance.getSnapshot().open; + const isOpen = () => tool === "logs" ? logs.getSnapshot().open : tool === "storage" ? storage.getSnapshot().open : savedVisible || performance.getSnapshot().open; const closeTools = () => { setSavedVisible(false); if (logs.getSnapshot().open) logs.toggle(); performance.hide(); + storage.hide(); void performance.disconnect(); }; - const showTool = (next: "logs" | "performance") => { + const showTool = (next: Tool) => { const startedAt = globalThis.performance.now(); if (next === tool && isOpen()) { closeTools(); return; } setUiSurface(next); setTool(next); setSavedVisible(false); + if (next !== "storage") storage.hide(); if (next === "logs") { performance.hide(); logs.show(); } - else { logs.hide(); performance.show(); } + else if (next === "performance") { logs.hide(); performance.show(); } + else { logs.hide(); performance.hide(); storage.show(); } markUiSurfaceReady(startedAt); }; const logsRef = usePanelRef(); @@ -82,21 +93,23 @@ export function Workspace({ logs, performance, recordingController, updates, aut if (!previous) return; if (size.inPixels <= collapsedToolSize + 1 && isOpen()) closeTools(); else if (size.inPixels > 80 && isOpen() === false) showTool(tool); - }}>{tool === "logs" && }{tool === "performance" && open && (savedVisible ?
: )}; + }}>{tool === "logs" && }{tool === "storage" && }{tool === "performance" && open && (savedVisible ?
: )}; // The sidebar keeps these controls under its simulators. Fullscreen, or no visible // platform, moves them above the tools. const bottomToolbar = !fullscreen && layout !== "none"; const menuSide = bottomToolbar ? "top" : "bottom"; + const ToolIcon = toolIcons[tool]; const toolbar = ; - const simulatorPanel = + const simulatorPanel = setSimulatorsCollapsed(size.inPixels < 1)}>
@@ -134,7 +135,9 @@ export function Workspace({ logs, performance, storage, recordingController, upd {bottomToolbar && toolbar}
; - const divider =