Allow typed arrays in the nativeSymbols table

docs-developer/CHANGELOG-formats.md

@@ -6,6 +6,19 @@ Note that this is not an exhaustive list. Processed profile format upgraders can
 
 ## Processed profile format
 
+### Version 72
+
+The columns of the native symbol table (`profile.shared.nativeSymbols`) can now optionally be stored as typed arrays, for profiles loaded from [JsonSlabs](https://github.com/mstange/json-slabs/) files (.jslb, .jslb.gz). Regular JS / JSON arrays are still accepted.
+
+The "function size not known" sentinel value has changed from `null` to `-1` in both representations (JSON and JSLB).
+
+The column types are as follows:
+
+- `libIndex` (`Int32Array`)
+- `address` (`Uint32Array`)
+- `name` (`Int32Array`)
+- `functionSize` (`Int32Array`)
+
 ### Version 71
 
 The frame table (`profile.shared.frameTable`) representation changed in such a way that all its columns can now be typed arrays when using [JsonSlabs](https://github.com/mstange/json-slabs/) profiles.

src/app-logic/constants.ts

@@ -12,7 +12,7 @@ export const GECKO_PROFILE_VERSION = 36;
 // The current version of the "processed" profile format.
 // Please don't forget to update the processed profile format changelog in
 // `docs-developer/CHANGELOG-formats.md`.
-export const PROCESSED_PROFILE_VERSION = 71;
+export const PROCESSED_PROFILE_VERSION = 72;
 
 // The following are the margin sizes for the left and right of the timeline. Independent
 // components need to share these values.

src/profile-logic/data-structures.ts

@@ -6,7 +6,11 @@ import {
   GECKO_PROFILE_VERSION,
   PROCESSED_PROFILE_VERSION,
 } from '../app-logic/constants';
-import { toUint8OrUint16Array, valuesFitInUint8 } from '../utils/typed-arrays';
+import {
+  toFloat64ArraySetNullToZero,
+  toUint8OrUint16Array,
+  valuesFitInUint8,
+} from '../utils/typed-arrays';
 
 import type {
   RawProfileSharedData,
@@ -20,7 +24,7 @@ import type {
   FuncTable,
   RawMarkerTable,
   ResourceTable,
-  NativeSymbolTable,
+  RawNativeSymbolTable,
   Profile,
   ExtensionTable,
   CategoryList,
@@ -30,13 +34,13 @@ import type {
   SourceLocationTable,
   IndexIntoFrameTable,
   IndexIntoFuncTable,
+  IndexIntoLibs,
   IndexIntoStackTable,
   IndexIntoStringTable,
   IndexIntoCategoryList,
   IndexIntoSubcategoryListForCategory,
   IndexIntoNativeSymbolTable,
   IndexIntoSourceLocationTable,
-  IndexIntoLibs,
   InnerWindowID,
   Address,
   Bytes,
@@ -208,6 +212,22 @@ export function finishRawSamplesTableBuilder(
   };
 }
 
+export function getRawMarkerTableBuilder(): RawMarkerTableBuilder {
+  return {
+    // Important!
+    // If modifying this structure, please update all callers of this function to ensure
+    // that they are pushing on correctly to the data structure. These pushes may not
+    // be caught by the type system.
+    data: [],
+    name: [],
+    startTime: [],
+    endTime: [],
+    phase: [],
+    category: [],
+    length: 0,
+  };
+}
+
 export function getRawMarkerTableBuilderFromExisting(
   markerTable: RawMarkerTable
 ): RawMarkerTableBuilder {
@@ -230,6 +250,19 @@ export function getRawMarkerTableBuilderFromExisting(
   return builder;
 }
 
+export function finishRawMarkerTableBuilder(
+  builder: RawMarkerTableBuilder
+): RawMarkerTable {
+  return {
+    ...builder,
+    // The nulls in these columns become zeros. This is fine: whether a marker's
+    // start / end time is meaningful is determined by its phase, and the times
+    // which are not used are allowed to be arbitrary values.
+    startTime: toFloat64ArraySetNullToZero(builder.startTime),
+    endTime: toFloat64ArraySetNullToZero(builder.endTime),
+  };
+}
+
 export function getRawStackTableBuilderWithExistingContents(
   existing: RawStackTable
 ): RawStackTableBuilder {
@@ -408,61 +441,66 @@ export function shallowCloneSourceLocationTable(
   };
 }
 
-export function shallowCloneNativeSymbolTable(
-  nativeSymbols: NativeSymbolTable
-): NativeSymbolTable {
+export type RawNativeSymbolTableBuilder = {
+  libIndex: IndexIntoLibs[];
+  address: Address[];
+  name: IndexIntoStringTable[];
+  functionSize: Array<Bytes | -1>;
+  length: number;
+};
+
+export function getRawNativeSymbolTableBuilder(): RawNativeSymbolTableBuilder {
   return {
     // Important!
     // If modifying this structure, please update all callers of this function to ensure
     // that they are pushing on correctly to the data structure. These pushes may not
     // be caught by the type system.
-    libIndex: nativeSymbols.libIndex.slice(),
-    address: nativeSymbols.address.slice(),
-    name: nativeSymbols.name.slice(),
-    functionSize: nativeSymbols.functionSize.slice(),
-    length: nativeSymbols.length,
+    libIndex: [],
+    address: [],
+    name: [],
+    functionSize: [],
+    length: 0,
   };
 }
 
-export function getEmptyResourceTable(): ResourceTable {
+export function getRawNativeSymbolTableBuilderWithExistingContents(
+  nativeSymbols: RawNativeSymbolTable
+): RawNativeSymbolTableBuilder {
   return {
     // Important!
     // If modifying this structure, please update all callers of this function to ensure
     // that they are pushing on correctly to the data structure. These pushes may not
     // be caught by the type system.
-    name: [],
-    host: [],
-    type: [],
-    length: 0,
+    libIndex: Array.from(nativeSymbols.libIndex),
+    address: Array.from(nativeSymbols.address),
+    name: Array.from(nativeSymbols.name),
+    functionSize: Array.from(nativeSymbols.functionSize),
+    length: nativeSymbols.length,
   };
 }
 
-export function getEmptyNativeSymbolTable(): NativeSymbolTable {
+export function finishRawNativeSymbolTableBuilder(
+  builder: RawNativeSymbolTableBuilder
+): RawNativeSymbolTable {
   return {
-    // Important!
-    // If modifying this structure, please update all callers of this function to ensure
-    // that they are pushing on correctly to the data structure. These pushes may not
-    // be caught by the type system.
-    libIndex: [],
-    address: [],
-    name: [],
-    functionSize: [],
-    length: 0,
+    libIndex: new Int32Array(builder.libIndex),
+    // Uint32Array, like frameTable.address, so that the two can be compared.
+    address: new Uint32Array(builder.address),
+    name: new Int32Array(builder.name),
+    functionSize: new Int32Array(builder.functionSize),
+    length: builder.length,
   };
 }
 
-export function getEmptyRawMarkerTable(): RawMarkerTableBuilder {
-  // Important!
-  // If modifying this structure, please update all callers of this function to ensure
-  // that they are pushing on correctly to the data structure. These pushes may not
-  // be caught by the type system.
+export function getEmptyResourceTable(): ResourceTable {
   return {
-    data: [],
+    // Important!
+    // If modifying this structure, please update all callers of this function to ensure
+    // that they are pushing on correctly to the data structure. These pushes may not
+    // be caught by the type system.
     name: [],
-    startTime: [],
-    endTime: [],
-    phase: [],
-    category: [],
+    host: [],
+    type: [],
     length: 0,
   };
 }
@@ -625,7 +663,7 @@ export function getEmptyThread(overrides?: Partial<RawThread>): RawThread {
     samples: finishRawSamplesTableBuilder(
       getRawSamplesTableBuilderWithEventDelay()
     ),
-    markers: getEmptyRawMarkerTable(),
+    markers: finishRawMarkerTableBuilder(getRawMarkerTableBuilder()),
   };
 
   return {
@@ -640,7 +678,9 @@ export function getEmptySharedData(): RawProfileSharedData {
     frameTable: finishRawFrameTableBuilder(getRawFrameTableBuilder()),
     funcTable: getEmptyFuncTable(),
     resourceTable: getEmptyResourceTable(),
-    nativeSymbols: getEmptyNativeSymbolTable(),
+    nativeSymbols: finishRawNativeSymbolTableBuilder(
+      getRawNativeSymbolTableBuilder()
+    ),
     sources: getEmptySourceTable(),
     stringArray: [],
     sourceLocationTable: getEmptySourceLocationTable(),

src/profile-logic/global-data-collector.ts

@@ -5,10 +5,11 @@
 import { StringTable } from '../utils/string-table';
 import {
   finishRawFrameTableBuilder,
+  finishRawNativeSymbolTableBuilder,
   finishRawStackTableBuilder,
   getRawFrameTableBuilder,
+  getRawNativeSymbolTableBuilder,
   getEmptyFuncTable,
-  getEmptyNativeSymbolTable,
   getEmptyResourceTable,
   getEmptySourceTable,
   getEmptySourceLocationTable,
@@ -25,7 +26,6 @@ import type {
   SourceTable,
   FuncTable,
   ResourceTable,
-  NativeSymbolTable,
   IndexIntoResourceTable,
   IndexIntoFuncTable,
   ExtensionTable,
@@ -36,6 +36,7 @@ import type {
 import { ResourceType } from 'firefox-profiler/types';
 import type {
   RawFrameTableBuilder,
+  RawNativeSymbolTableBuilder,
   RawStackTableBuilder,
 } from './data-structures';
 
@@ -57,7 +58,8 @@ export class GlobalDataCollector {
   _stackTableBuilder: RawStackTableBuilder = getRawStackTableBuilder();
   _funcTable: FuncTable = getEmptyFuncTable();
   _resourceTable: ResourceTable = getEmptyResourceTable();
-  _nativeSymbols: NativeSymbolTable = getEmptyNativeSymbolTable();
+  _nativeSymbols: RawNativeSymbolTableBuilder =
+    getRawNativeSymbolTableBuilder();
   _funcKeyToFuncIndex: Map<string, IndexIntoFuncTable> = new Map();
   _nativeSymbolKeyToNativeSymbolIndex: Map<string, IndexIntoNativeSymbolTable> =
     new Map();
@@ -273,7 +275,7 @@ export class GlobalDataCollector {
       this._nativeSymbols.libIndex[nativeSymbolIndex] = libIndex;
       this._nativeSymbols.address[nativeSymbolIndex] = address;
       this._nativeSymbols.name[nativeSymbolIndex] = name;
-      this._nativeSymbols.functionSize[nativeSymbolIndex] = functionSize;
+      this._nativeSymbols.functionSize[nativeSymbolIndex] = functionSize ?? -1;
       this._nativeSymbolKeyToNativeSymbolIndex.set(key, nativeSymbolIndex);
     }
     return nativeSymbolIndex;
@@ -299,7 +301,7 @@ export class GlobalDataCollector {
       frameTable: finishRawFrameTableBuilder(this._frameTable),
       funcTable: this._funcTable,
       resourceTable: this._resourceTable,
-      nativeSymbols: this._nativeSymbols,
+      nativeSymbols: finishRawNativeSymbolTableBuilder(this._nativeSymbols),
       stringArray: this._stringArray,
       sources: this._sources,
       sourceLocationTable: getEmptySourceLocationTable(),

src/profile-logic/import/chrome.ts

@@ -12,6 +12,7 @@ import { FrameFlag } from 'firefox-profiler/types';
 
 import {
   finishRawSamplesTableBuilder,
+  finishRawMarkerTableBuilder,
   getEmptyProfile,
   getEmptyThread,
   getRawSamplesTableBuilderWithEventDelay,
@@ -431,7 +432,6 @@ function getThreadInfo(
   nodeIdToStackId.set(undefined, null);
 
   const markers = getRawMarkerTableBuilderFromExisting(thread.markers);
-  thread.markers = markers;
 
   const threadInfo: ThreadInfo = {
     thread,
@@ -851,6 +851,7 @@ async function processTracingEvents(
 
   for (const [thread, threadInfo] of threadInfoByThread) {
     thread.samples = finishRawSamplesTableBuilder(threadInfo.samples);
+    thread.markers = finishRawMarkerTableBuilder(threadInfo.markers);
   }
 
   return profile;

src/profile-logic/import/simpleperf.ts

@@ -32,8 +32,10 @@ import {
   type RawStackTableBuilder,
   getRawSamplesTableBuilder,
   type RawSamplesTableBuilder,
-  getEmptyRawMarkerTable,
-  getEmptyNativeSymbolTable,
+  getRawMarkerTableBuilder,
+  finishRawMarkerTableBuilder,
+  getRawNativeSymbolTableBuilder,
+  finishRawNativeSymbolTableBuilder,
   getEmptySourceTable,
   getEmptySourceLocationTable,
 } from 'firefox-profiler/profile-logic/data-structures';
@@ -239,7 +241,9 @@ class FirefoxSharedData {
       frameTable: this.frameTable.toJson(),
       funcTable: this.funcTable.toJson(),
       resourceTable: this.resourceTable.toJson(),
-      nativeSymbols: getEmptyNativeSymbolTable(),
+      nativeSymbols: finishRawNativeSymbolTableBuilder(
+        getRawNativeSymbolTableBuilder()
+      ),
       sources: getEmptySourceTable(),
       stringArray: this.stringArray,
       sourceLocationTable: getEmptySourceLocationTable(),
@@ -292,7 +296,7 @@ class FirefoxThread {
       pid: this.pid.toString(),
       tid: this.tid,
       samples: finishRawSamplesTableBuilder(this.sampleTable),
-      markers: getEmptyRawMarkerTable(),
+      markers: finishRawMarkerTableBuilder(getRawMarkerTableBuilder()),
     };
   }
 

src/profile-logic/js-tracer.ts

@@ -3,7 +3,8 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 import {
   getRawSamplesTableBuilderWithEventDelay,
-  getEmptyRawMarkerTable,
+  getRawMarkerTableBuilder,
+  finishRawMarkerTableBuilder,
   finishRawFrameTableBuilder,
   finishRawSamplesTableBuilder,
   finishRawStackTableBuilder,
@@ -511,7 +512,7 @@ export function convertJsTracerToThreadWithoutSamples(
     weight: [],
     weightType: 'tracing-ms',
   };
-  const markers = getEmptyRawMarkerTable();
+  const markers = getRawMarkerTableBuilder();
 
   const thread: RawThread = {
     ...fromThread,
@@ -646,6 +647,7 @@ export function convertJsTracerToThreadWithoutSamples(
   shared.stackTable = finishRawStackTableBuilder(stackTable);
   shared.frameTable = finishRawFrameTableBuilder(frameTable);
   thread.samples = finishRawSamplesTableBuilder(samples);
+  thread.markers = finishRawMarkerTableBuilder(markers);
 
   return { thread, stackMap };
 }