diff --git a/backends/vulkan/test/ops/benchmark_v2/BUCK b/backends/vulkan/test/ops/benchmark_v2/BUCK new file mode 100644 index 00000000000..f559a6f1cfe --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/BUCK @@ -0,0 +1,9 @@ +# Any targets that should be shared between fbcode and xplat must be defined in +# targets.bzl. + +load("@fbsource//tools/build_defs:fbsource_utils.bzl", "is_fbcode") +load(":targets.bzl", "define_common_targets") + +oncall("executorch") + +define_common_targets(is_fbcode = is_fbcode()) diff --git a/backends/vulkan/test/ops/benchmark_v2/README.md b/backends/vulkan/test/ops/benchmark_v2/README.md new file mode 100644 index 00000000000..b745d451c54 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/README.md @@ -0,0 +1,995 @@ +# benchmark_v2: JSON shader and operator benchmarks + +`etvk_ops_bench` replays a registered ET-VK shader or operator from a JSON config. +Pass one object or an array mixing both case types. Each shader config or operator +sweep entry gets its own tensors, calibration, and measurements. Results default +to compact JSON on stdout, with argument strings and timing summaries. Runtime +failures appear as error objects and execution continues with the next config. +Use `--output_format=json_full` for full ordered JSON reports or +`--output_format=table` for a condensed table. + +## Layout + +Define inputs, tensor shapes, initialization, sweeps, and measurement settings in +JSON. The binary validates outputs against registered CPU references, calibrates +the repetition count, and reports GPU timestamps, wall time, and optional +throughput estimates. It executes shaders and operators linked into the ET-VK +runtime; raw shader configurations must match the selected shader's interface. + +Paths below are relative to `test/ops/benchmark_v2/`: + +```text +etvk_ops_bench.cpp CLI and case execution +framework/ Parsing, graph setup, validation, timing, and reporting + test/ Framework and registry tests +operators/ CPU references, routes, throughput counters, and checks + /.../cases/ JSON case collections by operator and quantization scheme +examples/ Small shader and operator configuration examples +``` + +Each expanded case runs separately. Shader variants can share one binding +interface through `shader_sweep`; operators can sweep argument sets, dtype and +storage variables, and execution routes. + +## Running the binary + +The commands below assume `etvk_ops_bench` is on `PATH` and the current directory +is `test/ops/benchmark_v2/`. A host run requires a Vulkan loader and a compute-capable +Vulkan implementation. Select a hardware GPU or configure a software implementation +such as SwiftShader in the execution environment. + +```sh +etvk_ops_bench --config=examples/shaders.json +etvk_ops_bench --config=examples/linear.json --output_format=table +etvk_ops_bench --config=examples/conv2d.json --validation_only --validation_mode=first +``` + +For Android, stage an Android build of the executable and the JSON file with ADB. +Replace `SERIAL` with the connected device's serial and `/path/to/etvk_ops_bench` +with the built executable: + +```sh +adb -s SERIAL push /path/to/etvk_ops_bench /data/local/tmp/etvk_ops_bench +adb -s SERIAL push examples/shaders.json /data/local/tmp/benchmark_config.json +adb -s SERIAL shell chmod +x /data/local/tmp/etvk_ops_bench +adb -s SERIAL shell /data/local/tmp/etvk_ops_bench --config=/data/local/tmp/benchmark_config.json +``` + +## Command-line options + +The binary requires `--config=`. All options follow the executable +name; tensor and benchmark settings belong in the JSON file. + +| Option | Default | Behavior | +| --- | --- | --- | +| `--config=` | Required | Read one case or an array of case definitions. | +| `--output_format=json\|json_full\|table` | `json` | Compact JSON, full JSON, or human-readable table. | +| `--validation_mode=all\|first\|none` | `all` | Validate every case, only the first expanded case, or no cases. | +| `--error_mode=report\|throw\|skip` | `report` | Report failures and continue, stop on the first failure, or disable validation. Unsupported cases report `SKIP`. | +| `--validate_specs` | `false` | Audit schemas and registrations without allocating case tensors or running GPU work. | +| `--validation_only` | `false` | Run correctness checks without calibration or timing passes. Requires validation enabled. | +| `--table_name_max_chars=` | `32` | Limit shader/operator names in table output; `0` disables truncation. | +| `--table_inputs_max_chars=` | `64` | Limit table input strings; `0` disables truncation. | +| `--helpshort` | — | Show the binary's flags and defaults. | + +For a quick correctness check, pass `--validation_only --validation_mode=first`. +For timing without CPU reference work, pass `--validation_mode=none`. +Benchmark conditions such as `target_ms` and `benchmark_runs` belong in the JSON. + +## Config + +| Example | Cases | +| --- | --- | +| `examples/buffer_shader.json` | Raw buffer copy. | +| `examples/texture_shader.json` | Raw 3D texture addition. | +| `examples/texture_shader_lwg_sweep.json` | Texture addition with explicit LWG candidates, including `[10,10,1]`. | +| `examples/texture_shader_lwg_range.json` | Texture addition with a Cartesian LWG range. | +| `examples/shader_workgroup_sweep.json` | Two compatible 32-bit buffer-copy shaders crossed with four LWG sizes. | +| `examples/shaders.json` | Raw buffer copy and texture addition. | +| `examples/conv2d.json` | FP32 and FP16 convolution. | +| `examples/conv2d_routes.json` | FP32/FP16 direct and im2col convolution, including intermediate-storage variants. | +| `examples/q8ta_conv2d_routes.json` | Q8TA automatic, direct, and im2col convolution. | +| `examples/q8ta_conv2d.json` | Quantized int8 activations, per-channel int8 weights, and int8 output. | +| `examples/linear.json` | FP32/FP16 linear projections and feed-forward expansion, with decode and batched/prefill shapes. | +| `examples/q4gsw_linear.json` | Weight-only Q4GSW linear with FP32 and FP16 activations. | + +| Field | Meaning | +| --- | --- | +| `shader` | Compiled registry name, e.g. `buffer_to_buffer_float`. | +| `operator` | Operator registry name, e.g. `aten.linear.default`. Specify exactly one of `shader`, `shader_sweep`, `operator`, or `check`. | +| `shader_sweep` | Nonempty shader-name list; every shader uses the shared bindings, initialization, parameters, and benchmark settings. | +| `name` | Optional result label; defaults to the registry name. | +| `gwg_size` | Three positive shader invocation counts or a global picker name. | +| `lwg_size` | Three positive workgroup dimensions or a local picker name. | +| `lwg_size_sweep` | Shader-only Cartesian search: `x`, `y`, and `z` ranges `[start, stop, step]`. | +| `bindings` | Shader bindings in binding order, with tensor properties declared inline. | +| `sweep_variables` | Operator variable names mapped to nonempty choice arrays; all Cartesian combinations are tested. | +| `route` | Operator-only route name or `{"name":"im2col","options":{...}}`; omitted means normal automatic routing. | +| `route_sweep` | Operator-only nonempty list of route selections, crossed with argument sets and variables. | +| `args_sweep` | Nonempty list of full operator argument arrays; each set is benchmarked separately. | +| `specialization_constants` | Optional array for IDs 3 onward; local dimensions occupy IDs 0–2. | +| `push_constants` | Optional array of typed data blocks, concatenated into at most 128 bytes. | +| `outputs` | Shader-only optional logical output names. | +| `target_ms` | Calibration target per batch, default 50. | +| `max_dispatches` | Invocation count cap, default 200 for operators and 65536 for shaders. | +| `warmup_runs` | Unmeasured batches per timing mode, default 3. | +| `benchmark_runs` | Measured batches per timing mode, default 10. | +| `check` | Registered validation-only check, e.g. capability or dynamic-resize checks. | +| `reference` | Optional registered CPU reference name; inferred for supported operators/shaders. | +| `validation_method` | Comparison function and tolerances, e.g. `{"function":"is_close","atol":0.01,"rtol":0.01}`. Optional `outputs` selects named tensors to check. | +| `show_memory_throughput` | Show registered memory-throughput estimates for each measurement; default false. | +| `show_compute_throughput` | Show registered compute-throughput estimates for each measurement; default false. | + +Tensor descriptions require `sizes` in logical NCHW order. For shaders, put these +properties inside the `buffer`, `texture2d`, or `texture3d` binding object; +for operators, use the `tensor` object inside each argument set in `args_sweep`. Optional fields: + +- `name`: optional tensor name; defaults to `arg_` + for operators and `binding_` for shaders. +- `dtype`: `float32` (default), `float16`, `int32`, `int8`, `uint8`, or `int8x4`. +- `storage`: operator-only; `buffer` (default), `texture2d`, `texture3d`, or + `constant`. Constant storage passes a CPU tensor reference to the operator's + prepack route; the operator chooses its GPU storage. + For shader bindings, the wrapper key determines storage. +- `layout`: `width` (default), `height`, or `channels` packing. +- `data`: initialization descriptor, described below; defaults to zero. +- `access`: optional operator role override, `read`, `write`, or `read_write`. + Roles are inferred from the operator's dispatch nodes when omitted. + Writable tensors cannot be constant. + +| `data` | Initialization | +| --- | --- | +| `{"fill":0.12}` | Fill every logical element with a value representable by the dtype. | +| `{"rand_uniform_float":[0,1]}` | Uniform random values for FP32/FP16 tensors. | +| `{"rand_uniform_float":[0,1],"seed":42}` | Uniform values using an explicit uint32 seed. | +| `{"rand_uniform_int":[-8,8]}` | Uniform integers, including both bounds; also supports integer-valued FP32/FP16 inputs. Optional `seed` is supported. | +| `{"repeat":[1,2,3]}` | Repeat a fixed sequence to fill the tensor. | +| `{"abs_mod":{"bounds":[-128,127],"mod":5}}` | Generate uniform integers, then take `abs(value) % mod`. | +| `{"pad_columns":{"source":{"rand_uniform_int":[-8,8]},"valid_columns":12}}` | Generate the entire padded tensor, then zero columns after the valid prefix. | +| `{"row_sum":"weight"}` | Sum flattened source rows into int32; trailing destination rows are zero. | +| `{"column_sum":"weight"}` | Sum the source's leading dimensions into its last dimension. | +| `{"q4_group_sum":{"tensor":"weight","group_size":32}}` | Sum signed packed Q4 weights per group into group-major `[K/group_size,N]` data. | + +Random data is reproducible. By default, each tensor name determines its seed, +so dtype/storage variants use the same source values and different tensors use +independent streams. FP16 conversion happens during tensor preparation. Flat +`data` arrays and a tensor-level `fill` field are also accepted; use either `data` +or `fill`. The tensor's `storage: "constant"` selects CPU prepacking inputs; +`data.fill` only controls initialization. +Example inputs use random initialization; `fill` is used only for output tensors. +`row_sum` resolves named tensor dependencies, including forward references, and +rejects missing sources, cycles, noninteger sources, and int32 overflow. + +For `int8x4`, use buffer storage and a packed layout: `4w`, `4c`, `4w4c`, +`4h4w`, or `4c1w`. Supply logical int8 values; GPU prepacking packs them into +int32 storage. Reports contain tensor properties and timing data without tensor contents. + +### Operator arguments + +Operators use `args_sweep`, a list of ordered argument arrays; workgroup dimensions and shader bindings +are selected by their runtime implementations. Arguments support: + +| JSON | Graph value | +| --- | --- | +| `{"tensor":{"name":"input","sizes":[4,32],"dtype":"float32"}}` | Inline tensor definition. Use `storage: "constant"` for a CPU tensor reference. | +| `{"tensor":{"ref":"input"}}` | Reuse an inline tensor definition. | +| `{"int":32}` | Integer. | +| `{"float":0.25}` | Floating point scalar. | +| `{"bool":false}` | Boolean. | +| `{"int_list":[1,1]}` | Integer list. | +| `{"string":"none"}` | String. | +| `null` | None, e.g. absent bias. | + +Each sweep entry contains the full registry argument list, including outputs, in signature order. +Arguments are passed directly without reordering or appending output arguments. +Output tensors do not require an `access` annotation. The runner infers input, +output, and read/write roles from the dispatch nodes for reporting. + +For example, each linear argument set has the order input, weight, bias, output. +To reuse an input as the output, pass `{"tensor":{"ref":"input"}}` at the output position. +References can precede definitions; names must be unique within a set and can +be reused in other sets. References cannot cross sweep entries. References accept `ref` +and optional `access`; other tensor properties belong in the definition. +Operator configs reject separate `tensors`, `args`, `inputs`, and `outputs` fields. +Weight and bias tensors usually require `storage: "constant"`. An operator +case cannot specify `gwg_size`, `lwg_size`, `bindings`, push constants, +or specialization constants. + +One case can benchmark both precisions: + +```json +{ + "name": "relu", + "operator": "aten.relu.default", + "target_ms": 10, + "benchmark_runs": 5, + "sweep_variables": {"v_DTYPE": ["float32", "float16"]}, + "args_sweep": [ + [ + {"tensor": {"sizes": [8], "dtype": "v_DTYPE", "data": {"rand_uniform_float": [0, 1]}}}, + {"tensor": {"sizes": [8], "dtype": "v_DTYPE", "data": {"fill": 0}}} + ] + ] +} +``` + +Case-level timing settings apply to every entry. Results follow definition and +sweep order, with separate calibration and measurements for each argument set. +Variable names start with `v_`. An entire matching string value is replaced by +the selected JSON value, including numbers, arrays, or objects. Variables can +appear in arguments or case-level settings. Choices must be concrete values; +undefined placeholders are rejected. For each argument template, variables are +expanded in lexical name order, with the last variable varying fastest. + +The linear example sweeps 1, 16, and 128 input rows for 2048→2048 projections +and 2048→8192 feed-forward expansion. Each shape runs in FP32 and FP16 with +buffer and texture3d activations, for 24 cases total. +Weights use uniform `[-0.02, 0.02]` values. Its explicit `atol=0.05`, `rtol=0.01` +allows FP16 accumulation error in long reductions; tighter tolerances can be used +when studying numerical accuracy. + +The convolution examples use 3x3 kernels, stride/dilation 1, and padding 1. +Q8TA weights use the flattened `[C_out, C_in * K_h * K_w]` layout. Weight sums +are derived from the random weights; the unsigned-dot path requires these sums +even when the input zero point is zero. Q4GSW weights use `[N, K/2]` uniform random +packed bytes; each byte stores two signed Q4 values with zero point 8. +Random scales remain positive and have shape `[K/group_size, N]`. + +### Operator routes + +Use `route_sweep` to compare execution routes with the same canonical +operator arguments and initialized data: + +```json +{ + "operator": "aten.convolution.default", + "route_sweep": [ + "auto", + "direct", + "im2col", + {"name": "im2col", "options": {"intermediate_storage": "buffer"}}, + {"name": "im2col", "options": {"intermediate_storage": "texture2d"}}, + {"name": "im2col", "options": {"intermediate_storage": "texture3d"}} + ], + "sweep_variables": {"v_DTYPE": ["float32", "float16"]}, + "args_sweep": [""] +} +``` + +For one selection, use `"route": "direct"` or an object with `name` +and optional `options`. The two fields are mutually exclusive and are rejected +for raw shaders. Variables can appear in route names and option values. Expansion +runs argument templates first, variable combinations next, and route +selections in listed order last. The combined expansion is capped at 65536 cases +per definition. Each case gets a separate graph, validation, calibration, and +measurements; route selectors are never appended to the operator argument list. + +`auto` calls the ordinary production operator registry, including its performance +heuristics. Forced routes call production route entry points directly, subject +to correctness and device-capability checks. A forced route never silently falls +back to another route. Validation uses the same selection in a separate +graph and the operator's existing CPU reference. + +Registered convolution routes: + +| Operator | Routes and options | +| --- | --- | +| `aten.convolution.default` | `auto`, `direct`, `im2col`. Only `im2col` accepts `intermediate_storage`: `buffer`, `texture2d`, or `texture3d`; omission uses the production choice. | +| `et_vk.q8ta_conv2d.default` | `auto`, `direct` (general convolution), `im2col`. No route options. | + +The floating-point forced routes currently support forward 2D convolution with +batch 1 and channels-packed texture3d input/output. Direct grouped convolution +requires group channel boundaries aligned to four; im2col requires groups 1. +The Q8TA routes use packed int8 buffers and the production streaming feasibility +checks. Route eligibility is separate from automatic performance heuristics: +for example, forcing floating-point im2col can test dilated convolutions even +when automatic routing prefers direct. Constants use `storage: "constant"`. Floating-point weight and bias constants +must share a declared dtype, whose effective GPU dtype matches the input/output +dtype; devices that promote FP16 storage to FP32 keep that promotion supported. + +Unsupported forced cases report `status: "SKIP"` and `skip_reason`, with no timing +measurements. They do not fail the run and are skipped in every `error_mode`, +including `throw`. Unknown route names, malformed options, and unexpected runtime +failures remain errors. `--validation_mode=none` skips correctness validation; +it does not disable route eligibility checks. Since unsupported cases produce no +nodes from which to infer tensor roles, their table row may include all arguments. + +Explicit selections add requested/resolved `route` metadata to JSON. +An `auto` result identifies the actual production route when an inspector is +registered; otherwise `resolved` is null. The table adds a `Route` +column, e.g. `auto->im2col` or `im2col[buffer]`. Shader breakdowns still show the +actual shaders, and the operator timing covers the complete execute chain. +Prepacking remains outside benchmark timing. + +### Registering routes + +`framework/RouteRegistry.h` registers a builder function directly under +an operator/route pair, plus an optional inspector for normal automatic routing: + +```cpp +REGISTER_OPERATOR_ROUTES { + REGISTER_OPERATOR_ROUTE(aten.my_op.default, direct, build_my_direct); + REGISTER_ROUTE_INSPECTOR(aten.my_op.default, inspect_my_route); +} +``` + +Builders live under `operators//`. Their signature is +`Json(ComputeGraph&, const std::vector&, const Json& options)`. +They receive the canonical argument list and return resolved metadata +`{"name":"direct","options":{}}`. Reject unknown options with +`std::invalid_argument`; throw `UnsupportedRoute` for expected shape, +storage, or capability restrictions before adding nodes. Preflight entry points +that can fall back internally so a forced result always describes the requested +route. The registry reserves `auto` for ordinary operator dispatch. + +An inspector receives the graph, canonical arguments, and the first/end execute +node indices for that operator. It returns resolved metadata or null when the +route cannot be identified. Route names are scoped to each operator; duplicate +operator/route registrations and null functions are rejected. Multiple pairs +can register the same builder function. The route library uses +whole-archive linking to retain static registrations. + +### Workgroup pickers + +Shader workgroup settings accept vectors or picker names, independently: + +```json +{ + "gwg_size": "default_pick_gwg", + "lwg_size": "default_pick_lwg" +} +``` + +Use `shader_sweep` instead of `shader` to test variants with the same binding +interface and tensor representation: + +```json +{ + "shader_sweep": ["buffer_to_buffer_float", "buffer_to_buffer_int32"], + "gwg_size": "default_pick_gwg", + "lwg_size_sweep": {"x": [32, 128, 32]} +} +``` + +Shader names run in listed order, with every workgroup candidate tested before +moving to the next shader. Each combination gets a fresh graph and its own +`sweep_index`, calibration, validation, and measurements. The combined Cartesian +sweep is capped at 65536 cases. `shader_sweep` also combines with an explicit LWG +list; omitting a workgroup sweep tests each shader once with the selected LWG. +Each shader is checked against the bindings when executed; incompatible variants +report case errors using the configured `error_mode`. + +The combined example keeps FP32 buffer inputs unchanged. Its int32 copy variant +loads and stores the same 32-bit words, preserving the floating-point bits. +The shared `reference: "copy"` checks identical logical outputs for both shaders. + +The global picker is `default_pick_gwg`. It uses the first tensor binding whose +`access` is `write` or `read_write`: logical element count for buffers, texture +extents for textures. Local pickers are `default_pick_lwg`, `pick_required_lwg`, +`pick_xy_square_lwg`, `pick_xz_square_lwg`, and `pick_120shape_lwg`. +`pick_required_lwg` needs a global grid with a wrapping hint, such as the default +buffer grid. Unsupported names produce a case error. + +For a shader, `lwg_size` also accepts a nonempty list of vectors: + +```json +"lwg_size": [[8, 8, 1], [10, 10, 1]] +``` + +Alternatively, omit `lwg_size` and specify ranges: + +```json +"lwg_size_sweep": {"x": [4, 12, 4], "y": [4, 12, 4], "z": [1, 1, 1]} +``` + +Ranges include the stop when reached by the step. Omitted axes default to 1. +The Cartesian order is x outermost, then y, then z; each candidate has its own +`sweep_index`, graph, calibration, validation, and measurements. A definition +can expand to at most 65536 cases. Invalid device configurations report case +errors; they are not silently filtered. `lwg_size` and `lwg_size_sweep` are +mutually exclusive. + +Explicit local dimensions must be positive and fit device limits. For a +linear buffer grid, an explicit or picked local size also sets the wrapping +geometry. Explicit global vectors are rounded up to local workgroup multiples, +with no output tile scaling. Results report resolved `gwg_size` and `lwg_size` +vectors and, when used, `gwg_picker` and `lwg_picker` names. +Shaders using generic linear buffer indexing require `lwg_size.z=1`; larger z +dimensions need a shader that explicitly handles those invocations. + +### Raw shader bindings + +Binding forms: + +```json +[ + {"buffer": {"name": "output", "sizes": [8], "access": "write"}}, + {"buffer": {"name": "input", "sizes": [8], "dtype": "float32", "data": {"rand_uniform_float": [-1, 1]}}}, + {"meta_ubo": "output", "field": "meta"}, + {"uniform": [ + {"dtype": "int32", "data": [8, 0, 0, 0]}, + {"dtype": "float32", "data": [0.5, 0, 0, 0]} + ]} +] +``` + +Tensor bindings use a `buffer`, `texture2d`, or `texture3d` wrapper. Put tensor +properties and `access` inside it, with `sizes` defining a new tensor. Omit +`storage` because the wrapper determines it. `name` is optional and defaults to +`binding_`; names must be unique. `meta_ubo`, specialization layout +constants, and `outputs` refer to these names. References can precede definitions. +Use `{"buffer":{"ref":"input","access":"read"}}` to bind the same buffer again +without allocating another resource. A reference's wrapper type must match its +tensor. Shader configs do not accept a separate `tensors` object. + +`access` defaults to `read`; `read_write` is also supported. `meta_ubo` fields are +`meta` (default; `BufferMetadata` or `TextureMetadata`), `sizes`, `strides`, +`numel`, and `logical_limits`. Bindings must match the shader signature. + +UBOs and push constants use arrays of typed blocks. Blocks are concatenated +without implicit padding: include the shader's alignment/padding explicitly. +Supported block dtypes: `float32`, `int32`, `uint32`, `int8`, and `uint8`. +Total byte size must be a multiple of four. + +Specialization constants use `{"dtype":"int32","value":8}` or +`{"layout":"input"}` for a tensor's hashed layout. Supported scalar dtypes are +`int32` (default), `uint32`, `float32`, and `bool`. + +The binary executes shaders and operators already linked into the runtime. +It does not compile GLSL or load external SPIR-V. Raw shader cases require +the selected shader's tensor layouts and parameters. Operator cases invoke +the operator's prepack and dispatch routes. + +## Output and timing + +The default output is compact JSON. Use `--output_format=table` for a condensed +table: + +```sh +etvk_ops_bench --config=examples/shaders.json --output_format=table +``` + +Columns show shader/operator name, resolved GWG/LWG, inputs, and mean/p90 GPU +timestamp latency in microseconds. Tensor prefixes indicate storage: `b` buffer, +`t2` texture2d, `t3` texture3d, and `c` CPU constant. Non-constant tensors append +their layout name after the dtype, e.g. `b[1,1,1,8]f32.width` or +`b[60,64,40,51]i8x4.4w4c`. Constants omit the layout suffix because the operator +determines their prepacked GPU layout, e.g. `c[128,576]i8`. +Shader inputs include `read` and `read_write` tensor bindings in binding order; +operator inputs follow `args` order and include constant tensors and `read_write` +arguments. Scalar, list, string, and null operator arguments are included in +signature order. Raw shader uniform and push constant values are included. +Floating-point arguments use three significant figures, with scientific notation +when needed. This also applies to floating-point list elements in `args_str`. +Output-only +tensors and UBOs are omitted. Dtype suffixes reflect actual precision. Operator +rows use `-` for workgroup sizes and report latency for the full execute chain. +Unsupported timestamps show `N/A`; failed cases include their error message. +Operator labels append the zero-based sweep index, e.g. `aten.linear.default [1]`. +Separators follow the header and separate cases; operator shader breakdowns stay +in the same group. Names default to a 32-character limit and Inputs to 64; +long values end in `...`. Configure `--table_name_max_chars` and +`--table_inputs_max_chars`; `0` disables the corresponding limit. JSON strings +remain complete. + +Use `--output_format=json_full` for full reports. Shader fields follow this order: + +Workgroup dimension values and tensor size arrays are printed on one line. +Other report containers retain their indentation. + +1. `case_name`, `shader_name`. +2. `gwg_size`, `lwg_size`: `{"method": "explicit", "value": [8, 1, 1]}` or + the picker name as `method`, with resolved dimensions as `value`. +3. `bindings`: named tensor/UBO descriptions with physical `binding_index`, + type, sizes, dtype, layout, and access. Shared bindings include `ref`. +4. `specialization_constants`: actual values keyed by constant ID (3 onward; + local dimensions are already shown in `lwg_size`). Layout-derived constants + also identify their tensor. Push constants follow when configured. +5. Run info: device, target duration, repetition cap/count, warmup and benchmark + runs, and timestamp availability/precision. +6. `gpu_timestamp_shader_us`: statistics with `min`, `max`, `mean`, `p50`, + `p90`, and `p95`. +7. `gpu_timestamp_batch_us`, `wall_time_shader_us`, `wall_time_batch_us`: + arrays with one value per measured batch, in execution order. + +Tensor values are excluded. Dtype metadata reports the actual dtype; +`requested_dtype` appears when promotion occurs, such as FP16 to FP32 on +SwiftShader. + +### JSON summaries (`json`) + +The default `--output_format=json` keeps the case grouping and field names of +full JSON, with these substitutions: + +- Operator `args` becomes `args_str`, using the full `Inputs` notation, including + non-tensor arguments, storage, dtype, and non-constant layouts. Output-only + tensors are excluded. Table width limits do not truncate this string. +- `gpu_timestamp_operator_us`, `gpu_timestamp_batch_us`, and wall timing arrays + become objects with `min`, `median`, and `max`. Median is the middle sorted + sample, averaging the two middle samples for even counts. Empty timing arrays + become `null`. +- Each `*_observed_gpu_clock_freq_MHz` array becomes `{min, max}` over that + pass's start/end readings. Unavailable readings are ignored; a pass without + usable readings becomes `null`. Zero remains a valid observation. + +Shader timestamp statistics, shader breakdowns, raw shader bindings, validation, +and other metadata retain their full JSON representation. + +```json +{ + "operator_name": "aten.linear.default", + "args_str": "b[4,32]f32.width, c[16,32]f32, c[16]f32", + "operator_invocations_per_batch": 10, + "gpu_timestamp_operator_us": {"min": 2, "median": 4.5, "max": 9}, + "gpu_timestamp_batch_us": {"min": 20, "median": 45, "max": 90}, + "gpu_batch_observed_gpu_clock_freq_MHz": {"min": 800, "max": 950} +} +``` + +### Batches and calibration + +A batch is one recorded command buffer containing repeated executions of the +case. For a raw shader, each repetition is one shader dispatch; for an operator, +each repetition is the full execute chain, which may dispatch several shaders. +The repetition count is reported as `dispatches_per_batch` or +`operator_invocations_per_batch`, respectively. + +Calibration always warms and probes query-free batches, increasing repetitions +until the measured wall time reaches roughly `target_ms` or `max_dispatches`. +Calibration probes are excluded from reported latency samples. Commands are +recorded once per calibration step, then reused for warmup and measurement. +Every expanded sweep combination is calibrated and measured separately. +Operator graphs always invoke resize functions before benchmarking, including +shader and workgroup selection even when input shapes have not changed. + +### Measurement passes + +The benchmark runs the enabled passes below in the listed order, using separate +submissions. + +| Pass | Measurement boundary | Reported latency | +| --- | --- | --- | +| Wall time | Host monotonic clock from command-buffer submission through fence completion, without timestamp queries. Includes host submission overhead, queue/fence waiting, and GPU execution. | `wall_time_batch_us` contains batch totals. Dividing each total by the repetition count gives `wall_time_shader_us` or `wall_time_operator_us`. | +| Whole-batch GPU timestamps | One timestamp pair brackets the entire repeated dispatch chain, including GPU dependency barriers and inter-dispatch gaps. | `gpu_timestamp_batch_us` describes batch totals. Dividing each total by the repetition count gives `gpu_timestamp_shader_us` or `gpu_timestamp_operator_us`. These are two views of the same capture. | +| Per-dispatch GPU timestamps | Operator-only profiling pass with a timestamp pair around each executed shader, including its associated barriers. | `shader_breakdown[].gpu_timestamp_shader_us` describes each dispatch position, averaged across that profiling batch's operator repetitions. | + +Pass selection depends on case type and timestamp support: + +| Case | GPU timestamps available | Enabled passes | +| --- | --- | --- | +| Raw shader | Yes | Wall time, whole-batch GPU timestamps. | +| Operator | Yes | Wall time, whole-batch GPU timestamps, per-dispatch GPU timestamps. | +| Raw shader or operator | No | Wall time only. GPU batch/operator timings are `null` in `json` and empty arrays in `json_full`; raw shader statistics are `null`, and shader breakdowns are omitted. | + +Each enabled pass performs `warmup_runs` unmeasured replays (default 3), followed +by `benchmark_runs` measured replays (default 10; many examples specify 5). +The wall and whole-batch GPU passes use the calibrated repetition count. The +profiling pass may reduce that count to fit its timestamp query pool; its divisor +is `shader_profile_invocations_per_batch`. + +### Android GPU clock samples + +Android builds query the GPU clock immediately before submission and after fence +completion for every measured replay. Reads are outside the latency intervals; +warmups, calibration, and validation do not produce clock samples. JSON reports +include separate clock series for each pass. `json` summarizes each series as +`{min, max}`; `json_full` preserves arrays of start/end pairs in MHz: + +```json +{ + "gpu_batch_observed_gpu_clock_freq_MHz": [ + [903, 903], + [903, 950] + ], + "wall_time_batch_observed_gpu_clock_freq_MHz": [ + [903, 903], + [903, 903] + ] +} +``` + +| Clock field | Measurement pass | Associated timing | +| --- | --- | --- | +| `gpu_batch_observed_gpu_clock_freq_MHz` | Whole-batch GPU timestamps | `gpu_timestamp_batch_us` and operator batch averages in `gpu_timestamp_operator_us`. | +| `wall_time_batch_observed_gpu_clock_freq_MHz` | Wall time | `wall_time_batch_us` and `wall_time_shader_us`/`wall_time_operator_us`. | +| `shader_profile_batch_observed_gpu_clock_freq_MHz` | Operator shader profiling | Runs summarized by `shader_breakdown[].gpu_timestamp_shader_us`. | + +In `json_full`, each enabled pass contributes `benchmark_runs` pairs in run +order. For GPU-batch and wall-time arrays, entry `i` corresponds to timing sample +`i` from that pass. +The GPU-batch and wall-time clock arrays immediately follow their corresponding +`_batch_us` fields in JSON output. Array indexing applies to `json_full`; compact +`json` reports extrema over each pass's observations. +Shader-profile entry `i` describes profiling batch `i`, whose dispatch latencies +feed the shader-breakdown statistics. The GPU batch array is empty without GPU +timestamp support; the shader profiling array appears only when operator +profiling runs. + +An unreadable clock sample is `null`; a driver-reported zero is retained. These +are driver-reported snapshots, not average clocks during GPU execution. Host +builds omit clock fields. + +When clock observations are present, tables add `Min GPU MHz` and `Max GPU MHz` +after the latency columns. A main case row reports the minimum and maximum +across all start/end observations from its wall, GPU batch, and shader profiling +runs. Indented shader rows share the range from the operator's profiling runs. +Missing, nonnumeric, or nonfinite readings are ignored; rows without usable +observations show `N/A`. Driver-reported zero values remain valid observations. + +The sampler discovers readable KGSL/devfreq, Pixel Mali, and Samsung SGPU sysfs +sources and converts their driver-specific Hz, kHz, or MHz units to MHz. Clock +access is best effort and does not require root or change GPU clock settings. + +### Samples and percentiles + +Each measured batch contributes one sample to each reported latency series. +For whole-batch GPU timing, the sample is the batch timestamp interval. For +amortized shader/operator timing, it is that interval divided by the repetition +count. For a shader breakdown entry, it is the average interval for that dispatch +position across the profiling batch's repetitions. Repeated shader names at +different dispatch positions remain separate series. + +In `json_full`, `gpu_timestamp_batch_us`, `gpu_timestamp_operator_us`, and +wall-time fields contain arrays with one value per measured batch, in execution +order. Compact `json` summarizes these arrays as `min`, `median`, and `max`. +Operator GPU values are batch totals divided by +`operator_invocations_per_batch`; for example, in `json_full`: + +```json +{ + "operator_invocations_per_batch": 10, + "gpu_timestamp_operator_us": [3, 1, 2], + "gpu_timestamp_batch_us": [30, 10, 20] +} +``` + +`gpu_timestamp_shader_us`, both for raw shaders and shader breakdown entries, +reports `min`, `max`, `mean`, `p50`, `p90`, and `p95`. These shader statistics +currently use one batch-average sample per benchmark run. The table derives +mean/p90 from operator sample arrays and reads shader statistics for raw shader +and indented profiling rows. + +Percentiles use linear interpolation. For `R` sorted samples `x[0]` through +`x[R-1]` and percentile fraction `f`, the position is `f * (R - 1)`; the result +interpolates between the adjacent samples at that position. With five sorted +samples `x1 <= x2 <= x3 <= x4 <= x5`: + +| Statistic | Result | +| --- | --- | +| p50 | `x3` | +| p90 | `0.4 * x4 + 0.6 * x5` | +| p95 | `0.2 * x4 + 0.8 * x5` | + +With `benchmark_runs: 5`, even a batch containing hundreds of dispatches yields +only five batch-average samples. High percentiles then largely interpolate +toward the largest observation and provide weak estimates of the tail. +Increasing `benchmark_runs` to 50 or 100 improves sampling, but these remain +percentiles of batch-average latency; averaging within each batch smooths +variation among individual executions. + +### Measurement boundaries and shader breakdowns + +Wall timing samples cover query-free submission through fence completion. GPU +timestamps bracket the repeated dispatch chain, including dependency barriers. Input +preparation, uploads, pipeline creation, command recording, query retrieval, and +readback are outside these intervals. Unsupported timestamps produce `null` GPU +batch/operator timings in `json`, empty arrays in `json_full`, and `null` raw +shader statistics. Conversion preserves fractional `timestampPeriod` and handles +timestamp counter width. + +Operator reports use `operator_name`, `sweep_index`, and the selected inputs: +`args_str` in `json`, or ordered `args` with inline tensor metadata and access +roles in `json_full`. They also include +the runtime-selected `execute_nodes`. Per-operator fields are +`gpu_timestamp_operator_us` and `wall_time_operator_us`; they include the full +execute chain (e.g. im2col, transpose, GEMM), with weight prepacking outside timing. +`operator_invocations_per_batch` is the divisor for both wall and batch GPU +measurements. Nodes may self-gate and issue +no dispatch. Repeats reuse initialized storage, so in-place operators accumulate +updates. +When variables are present, reports include their selected `sweep_variables` +values and the original `args_sweep_index`. `sweep_index` identifies the expanded +run within that definition. + +When GPU timestamps are available, operator reports also include `shader_breakdown`. +Each entry describes an actual dispatch in execution order: `dispatch_index`, +runtime-selected `shader_name`, resolved GWG/LWG, and `gpu_timestamp_shader_us` +statistics. Repeated shader names remain separate entries; skipped nodes are +omitted. Table output shows these shaders indented below their operator. + +The breakdown uses a separate profiling pass with the calibrated invocation count, +capped by timestamp query-pool capacity. `shader_profile_invocations_per_batch` +reports that count. Warmup and benchmark run counts are the same as the aggregate +timing passes. Each shader sample averages that dispatch position over the batch +and includes its dependency barrier. +Per-dispatch timestamp overhead and inter-dispatch gaps mean the sum need not +match the separately measured operator latency. The passes can also run under +different GPU clock/cache conditions, and their repetition counts may differ. +For small workloads, per-dispatch timestamps can substantially increase measured +latency. Compare the batch GPU timestamps and query-free wall measurements +when evaluating that overhead. +Prepacking remains outside timing. +Breakdowns are omitted when timestamps are unavailable, including on this +repository's SwiftShader build. + +## Throughput + +Enable either or both metrics in a case definition: + +```json +{ + "show_memory_throughput": true, + "show_compute_throughput": true +} +``` + +The report's `throughput` object contains `memory_bytes` and/or `compute_flops` +per invocation. `gpu_timestamp` and `gpu_timestamp_batch` contain rate statistics; +`wall_time` and `wall_time_batch` contain one rate per measured batch. Memory +rates are available as `memory_gib_per_s` (GiB/s, 2^30 bytes/s) and +`memory_mib_per_s` (MiB/s, 2^20 bytes/s); compute rates use `compute_gflops` +(10^9 FLOPs/s). Throughput is computed from each sample before aggregation, so +its percentiles rank rates, with higher values indicating faster execution. +Batch rates account for all repeated invocations and match the invocation rates. + +Table output appends `GPU GiB/s` and/or `GPU GFLOPS`, showing mean GPU rates. +Shader breakdown entries include their own GPU throughput when a counter is +registered for the executed shader. No parent operator work is assigned to a +child shader. Unregistered counters produce null rates, table `N/A`, and a +diagnostic; a known zero-work counter produces zero. Unavailable GPU timestamps +produce null GPU rates while wall-time rates remain available. + +Memory counters estimate logical tensor traffic: each distinct read tensor once, +each write tensor once, and read/write tensors twice. Operator counters include +logical input/weight/bias/output tensors; child shader counters estimate their bound tensor footprints, including packed +weights and intermediates. Tiled/streaming dispatches may touch only part of those +footprints. The model excludes +UBOs, staging, tensor padding, prepacking, and additional/cache-dependent traffic. +GEMM/convolution counters use two operations per multiply-accumulate, plus +optional bias additions. Quantized operators report arithmetic-equivalent work, +excluding weight dequantization and activation requantization. Q4 linear +counters can use the model `3*M*N*K + M*N`; FPA counters use the two-operation +multiply-accumulate model. The registered counter determines the model. Custom +counters can implement another work model. + +Counters register beside operator implementations using +`framework/ThroughputRegistry.h`, with independent memory/compute associations: + +```cpp +REGISTER_THROUGHPUT_FUNCTIONS { + REGISTER_OPERATOR_MEMORY_TRAFFIC(aten.my_op.default, my_byte_count); + REGISTER_OPERATOR_COMPUTE_FLOPS(aten.my_op.default, my_flop_count); + REGISTER_SHADER_MEMORY_TRAFFIC(my_shader, my_byte_count); + REGISTER_SHADER_COMPUTE_FLOPS(my_shader, my_flop_count); +} +``` + +Shader counters can register a case-sensitive prefix. For example: + +```cpp +REGISTER_SHADER_MEMORY_TRAFFIC(conv2d, throughput::tensor_traffic); +``` + +This covers every shader whose name starts with `conv2d`, including dtype, +storage, and tile variants. Lookup checks exact registrations first, then uses +the longest matching prefix. `REGISTER_SHADER_MEMORY` and +`REGISTER_SHADER_COMPUTE` register exact names for individual overrides; +operator associations remain exact. Memory and compute prefixes are independent, +and empty/duplicate prefix registrations and null callables are rejected. Each +registration stores the function pointer directly; one callable can serve +multiple operators, shaders, or prefixes. + +Prefixes group variants with the same work model. For example, `linear_vec_bias` +selects the bias-aware compute counter over `linear_vec`, and +`q4gsw_linear_gemm__tin__` selects its binding-aware memory counter over +`q4gsw_linear_gemm__`. Prefix association happens in the benchmark's throughput +registry when the actual dispatched shader name is reported. + +A counter returns a finite, nonnegative `double` from a `const Json&` containing +resolved case metadata. Operators receive `operator_name` and ordered `args` +with tensor descriptions and scalar values. Shaders receive `shader_name`, +resolved GWG/LWG, and named `bindings` with sizes, actual dtype, storage, and +access; shader breakdown counters receive metadata from that dispatch's tensors. +Registration is performed before benchmarking. Duplicate keys in each +association category are rejected. Functions are passed directly to registration +macros and receive resolved case metadata; no separate function-name registry +is required. + +## Correctness validation + +Validation runs before timing by default. It constructs a separate graph with +input/output staging and executes the target shader or operator once. Benchmark +graphs have no execute-stage input/output transfers. CPU reference work and +readback are outside all benchmark intervals. +Tensor data is generated once per expanded case and shared by the validation +and benchmark graphs. Each graph uploads its inputs during setup; benchmarking +does not regenerate input data. When validation is skipped, data is still +generated once to initialize the benchmark graph. +Validation GPU execution latency is not reported. The separate +`validation.reference_wall_time_us` field times CPU reference computation, +including input decoding and precision conversion; it excludes output readback +and comparison and is not a shader/operator benchmark latency. + +Select correctness checks with `--validation_mode`: + +| Value | Behavior | +| --- | --- | +| `all` (default) | Validate every expanded case when supported. | +| `first` | Validate only the first expanded case in the entire input file, for smoke testing. | +| `none` | Skip correctness validation and CPU reference computation. | + +`first` selects the first case after argument, shader, workgroup, and route sweep +expansion. If that case is unsupported, it reports `SKIP`; later cases are not +validated. All cases still run their benchmark passes when supported. Results +include a JSON `validation` object only for selected checks; the table includes +a `Validation` column when any result has validation information. + +`--error_mode=report` (default) reports `PASS` or `FAIL` for selected validation +checks and continues benchmarking. Any validation or execution failure produces +exit status 1. +`--error_mode=throw` stops at the first validation or execution failure with +exit status 1. `--error_mode=skip` disables validation, overriding +`--validation_mode`; prefer `--validation_mode=none`. +Malformed configs and flags produce exit status 2. + +Example reference names include `copy`, `add`, `linear`, `conv2d`, `q8ta_conv2d`, +and `q4gsw_linear`. They are inferred for the example shaders/operators; +`reference` can select one explicitly for a compatible custom shader. Unknown +references report validation failure. New CPU references belong under +`operators//` and register beside their implementations. + +### Registering references + +`framework/ReferenceRegistry.h` provides two registration levels: + +```cpp +REGISTER_REFERENCE_FUNCTIONS { + REGISTER_REFERENCE_FN(my_reference, my_reference_impl); + REGISTER_OPERATOR_REFERENCE(aten.my_op.default, my_reference); + REGISTER_SHADER_REFERENCE(my_shader, my_reference); +} +``` + +The first macro registers a callable under a unique name. The remaining macros +associate an operator or shader name with that reference name. Multiple names +may share a reference; operator and shader associations have separate namespaces. +Registration blocks run at static initialization, following the ET-VK operator +registration pattern. Associations may be declared before the function's +registration in another translation unit; lookup verifies that the function +exists. Duplicate names or associations are errors. +Each associated function must understand that case's operator arguments or +shader bindings. Registration is performed before benchmarking starts. + +A reference callable has the signature +`reference::Tensor(const Json&, const reference::TensorMap&, size_t threads)`. +Tensor inputs have already been decoded and rounded to actual GPU precision. +The returned tensor is shape-checked and rounded to output precision by the +framework. A JSON `reference` override resolves through the same function registry. +The reference library uses whole-archive linking to retain registration +initializers from every reference source file. + +Linear references use a tiled, vectorizable CPU GEMM with up to eight threads. +Convolution references parallelize output work. Validation does not skip cases +based on tensor size. Inputs are rounded to their actual GPU precision, +including FP16 constants, before computing reference results. + +The supported comparison is `is_close`: every element must satisfy +`abs(actual - expected) <= atol + rtol * abs(expected)`. Nonfinite values, +shape/count mismatches, and missing/unexpected outputs fail validation. +Defaults are FP32 `atol=rtol=1e-4`, FP16 `atol=rtol=1e-2`, and exact integer +comparison. `validation_method` overrides either tolerance; tolerances must be +finite and nonnegative. FP16 accumulation may require larger tolerances for +long reductions. Reports include per-output mismatch counts and error maxima, +and `reference_wall_time_us`, without tensor values. + +## Operator source organization + +Operator code follows family and quantization-scheme folders, using the production +runtime organization as a guide: + +```text +operators/ + convolution/ + Conv2dReference.{h,cpp} + Conv2dRoutes.{h,cpp} + Conv2dThroughput.cpp + cases/ + q8ta/ + Q8taConv2dReference.{h,cpp} + Q8taConv2dRoutes.{h,cpp} + Q8taConv2dThroughput.cpp + cases/ + gemm/ + linear/ + LinearReference.{h,cpp} + LinearThroughput.cpp + cases/ + qw/q4gsw/ + Q4gswLinearReference.{h,cpp} + Q4gswLinearThroughput.cpp + cases/ +``` + +Scheme-specific arithmetic, argument validation, and registrations live with the +scheme. Shared convolution geometry/traversal and route guards live in the +convolution parent; shared GEMM projection and work-count helpers live in the +GEMM parent. Add other quantized implementations under their family’s `q8ta/` or +`qw//` folder. + +## Framework components + +Shared headers and implementations live in `framework/`: `Config` validates JSON, +`Sweep` expands templates, Cartesian variables, and route selections, `DataInitializer` generates +typed input data, and `TensorUtils` handles tensor setup and dtype metadata. +`Timestamps` owns GPU queries, `Timing` handles calibration and measurement, +`WorkgroupUtils` resolves workgroup vectors and named pickers, and `Report` +formats ordered case metadata and timing statistics. +`RouteRegistry` resolves named operator routes and optional automatic +route inspectors. `ShaderBenchmark` and `OperatorBenchmark` assemble and run their respective cases. +Component tests live under `framework/test/`, including registry, reference, +route, reporting, and validation tests. +`etvk_ops_bench.cpp` handles the CLI and dispatches configs to the runners. + +## Framework tests + +Tests under `framework/test/` cover schemas and sweep expansion, data +initialization, reference and throughput registrations, route lookup, validation, +and report formatting. Registry and reporting tests run without GPU execution. +Operator route tests exercise shader chains, route options, automatic route +inspection, and unsupported-case guards; these require a Vulkan implementation +such as SwiftShader or a hardware GPU. + +## Case collections + +JSON collections live under `operators//.../cases/*.json`, alongside the +references and registrations for that operator family or quantization scheme. +Pass any collection to `--config`. Each definition and its expanded sweeps run +in file order. + +Operator cases call registered production operators directly. Q8TA cases use +packed `int8x4` activations and outputs; weight-only quantized operators use their +native weight, scale, and activation arguments. Quantize and dequantize are +separate operator cases. Quantized comparisons use quantized units with zero +relative tolerance. Use `route` or `route_sweep` to select execution routes. + +## Schema audits and validation-only runs + +`--validate_specs` parses all expanded cases and checks operator, route, reference, +and custom-check registrations without allocating case tensors or executing +operators. It emits a JSON audit summary and exits nonzero on errors. The binary +attempts texture creation without screening dimensions against driver-reported limits, +which can understate usable dimensions. Ordinary execution still validates +outputs according to `--validation_mode`. Use +`--validation_only` to execute GPU correctness checks without calibration or +benchmark passes; it requires `--validation_mode=all` or `first` and +`--error_mode=report` or `throw`. With `first`, remaining cases report `SKIP` +without allocating tensors or executing benchmark passes. + +Device support is determined while constructing and executing each case. The +binary catches ET-VK `ShaderNotSupportedError` and unsupported-route exceptions +and reports `SKIP` with a reason, then continues to the next case. This applies +to validation and benchmarking, including `--error_mode=throw`; that option +still stops on validation failures and other errors. JSON `requirements` gates +are not supported. `--validate_specs` checks schemas and registrations, rather +than executing cases to establish device support. + +Validation-only `check` cases return PASS, FAIL, or SKIP with diagnostic details; +they have no latency measurements. Unselected checks report `SKIP`, including +all checks under `--validation_mode=none` or `--error_mode=skip`. + +To preserve a case's declared validation scope when an operator also updates +auxiliary state, select outputs inside the comparison settings: + +```json +"validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001, + "outputs": ["output"] +} +``` + +The default checks all inferred writable tensors. An explicit list must be +nonempty, unique, and name writable GPU tensors from the case. Each selected +tensor must be produced by the reference and the GPU. Unselected auxiliary +tensors remain in the operator arguments and execute normally. diff --git a/backends/vulkan/test/ops/benchmark_v2/etvk_ops_bench.cpp b/backends/vulkan/test/ops/benchmark_v2/etvk_ops_bench.cpp new file mode 100644 index 00000000000..4d9f8ed8399 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/etvk_ops_bench.cpp @@ -0,0 +1,287 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include + +DEFINE_string( + config, + "", + "JSON file containing shader/operator configs, singly or as an array."); +DEFINE_bool( + validate_specs, + false, + "Validate case schemas and registry bindings without executing operators."); +DEFINE_bool( + validation_only, + false, + "Execute correctness checks without calibration or latency measurements."); +DEFINE_string( + validation_mode, + "all", + "Cases to validate: all, first, or none."); +DEFINE_string( + output_format, + "json", + "Output format: json (default), json_full, or table."); +DEFINE_string( + error_mode, + "report", + "Error handling: report or throw; skip disables validation for compatibility."); +DEFINE_uint32( + table_name_max_chars, + 32, + "Table name width limit; 0 means unlimited."); +DEFINE_uint32( + table_inputs_max_chars, + 64, + "Table Inputs width limit; 0 means unlimited."); + +namespace { + +vkcompute::shader_profiling::Report skip_report( + const vkcompute::shader_profiling::Json& spec, + const std::string& reason, + bool validation_enabled) { + using namespace vkcompute::shader_profiling; + Json accesses = Json::object(); + for (const auto& tensor : spec.at("tensors").items()) { + accesses[tensor.key()] = "unknown"; + } + Report result = spec.contains("operator") + ? operator_report(spec, Json::object(), accesses) + : shader_report(spec, Json(), Json(), Json::object(), Report::array()); + result["status"] = "SKIP"; + result["skip_reason"] = reason; + if (validation_enabled) { + result["validation"] = {{"status", "SKIP"}}; + } + Report throughput = error_report(spec, reason); + if (throughput.contains("throughput")) { + result["throughput"] = std::move(throughput["throughput"]); + } + return result; +} + +} // namespace + +int main(int argc, char* argv[]) { + gflags::SetUsageMessage( + "Replay registered ET-VK shaders or operators from --config=."); + gflags::ParseCommandLineFlags(&argc, &argv, true); + if (FLAGS_config.empty() || argc != 1) { + std::cerr << "Usage: etvk_ops_bench --config= " + "[--output_format=json|json_full|table] " + "[--validation_mode=all|first|none]\n"; + return 2; + } + using namespace vkcompute::shader_profiling; + try { + if (FLAGS_output_format != "json" && FLAGS_output_format != "table" && + FLAGS_output_format != "json_full") { + throw std::invalid_argument( + "output_format must be json, json_full, or table"); + } + if (FLAGS_validation_mode != "all" && FLAGS_validation_mode != "first" && + FLAGS_validation_mode != "none") { + throw std::invalid_argument( + "validation_mode must be all, first, or none"); + } + if (FLAGS_error_mode != "report" && FLAGS_error_mode != "throw" && + FLAGS_error_mode != "skip") { + throw std::invalid_argument("error_mode must be report, throw, or skip"); + } + if (FLAGS_validation_only && + (FLAGS_validation_mode == "none" || FLAGS_error_mode == "skip")) { + throw std::invalid_argument( + "validation_only requires validation_mode=all or first and " + "error_mode=report or throw"); + } + std::ifstream file(FLAGS_config); + if (!file) { + throw std::invalid_argument("Cannot open config: " + FLAGS_config); + } + Json document; + file >> document; + const auto configs = parse_configs(document); + if (FLAGS_validate_specs) { + Report errors = Report::array(); + for (const auto& config : configs) { + try { + if (config.contains("check")) { + case_check_registry().get_check(config.at("check")); + } else if (config.contains("operator")) { + const std::string op = config.at("operator"); + VK_GET_OP_FN(op); + if (config.contains("route") && + config.at("route").at("name") != "auto") { + route_registry().operator_route( + op, config.at("route").at("name")); + } + const auto reference = reference_name(config); + if (!reference_registry().has_reference(reference)) + throw std::invalid_argument( + "Unknown reference function: " + reference); + } + } catch (const std::exception& error) { + errors.push_back( + {{"case_name", config.value("name", "")}, + {"error", error.what()}}); + } + } + const Report summary{ + {"status", errors.empty() ? "PASS" : "FAIL"}, + {"case_count", configs.size()}, + {"errors", errors}}; + std::cout << summary.dump(2) << std::endl; + return errors.empty() ? 0 : 1; + } + Report results = Report::array(); + bool failed = false; + for (size_t case_index = 0; case_index < configs.size(); ++case_index) { + const Json& config = configs.at(case_index); + const bool validation_enabled = FLAGS_error_mode != "skip" && + (FLAGS_validation_mode == "all" || + (FLAGS_validation_mode == "first" && case_index == 0)); + if (config.contains("check")) { + Report result; + if (!validation_enabled) { + result = { + {"case_name", + config.value("name", config.at("check").get())}, + {"check_name", config.at("check")}, + {"status", "SKIP"}, + {"skip_reason", + FLAGS_error_mode == "skip" + ? "Validation-only case; error_mode=skip" + : "Validation-only case; validation_mode=" + + FLAGS_validation_mode}}; + } else { + result = run_case_check(config); + if (result.at("validation").at("status") == "FAIL") { + failed = true; + if (FLAGS_error_mode == "throw") { + std::cerr << result.at("validation").dump() << std::endl; + return 1; + } + } + } + results.push_back(std::move(result)); + continue; + } + if (FLAGS_validation_only && !validation_enabled) { + results.push_back(skip_report( + config, + "Validation not selected by validation_mode=" + + FLAGS_validation_mode, + true)); + continue; + } + Report validation; + Report validation_result; + TensorData host_data; + try { + host_data = prepare_tensor_data(config); + } catch (const std::exception& e) { + failed = true; + if (FLAGS_error_mode == "throw") { + std::cerr << e.what() << std::endl; + return 1; + } + Report result = error_report(config, e.what()); + if (validation_enabled) { + result["validation"] = {{"status", "FAIL"}, {"error", e.what()}}; + } + results.push_back(std::move(result)); + continue; + } + if (validation_enabled) { + try { + validation_result = config.contains("operator") + ? benchmark_operator(config, host_data, true) + : benchmark_shader(config, host_data, true); + validation = validation_result.at("validation"); + } catch (const UnsupportedRoute& e) { + results.push_back(skip_report(config, e.what(), validation_enabled)); + continue; + } catch (const vkcompute::vkapi::ShaderNotSupportedError& e) { + results.push_back(skip_report(config, e.what(), validation_enabled)); + continue; + } catch (const std::exception& e) { + validation = {{"status", "FAIL"}, {"error", e.what()}}; + } + if (validation.at("status") == "FAIL") { + failed = true; + if (FLAGS_error_mode == "throw") { + std::cerr << "Validation failed for " + << config.value( + "name", + config.value( + "operator", config.value("shader", ""))) + << ": " << validation.dump() << std::endl; + return 1; + } + } + } + if (FLAGS_validation_only) { + if (validation_result.is_null()) { + validation_result = error_report(config, validation.dump()); + } + validation_result["validation"] = std::move(validation); + results.push_back(std::move(validation_result)); + continue; + } + Report result; + try { + result = config.contains("operator") + ? benchmark_operator(config, host_data) + : benchmark_shader(config, host_data); + } catch (const UnsupportedRoute& e) { + results.push_back(skip_report(config, e.what(), validation_enabled)); + continue; + } catch (const vkcompute::vkapi::ShaderNotSupportedError& e) { + results.push_back(skip_report(config, e.what(), validation_enabled)); + continue; + } catch (const std::exception& e) { + failed = true; + if (FLAGS_error_mode == "throw") { + std::cerr << e.what() << std::endl; + return 1; + } + result = error_report(config, e.what()); + } + if (!validation.is_null()) { + result["validation"] = std::move(validation); + } + results.push_back(std::move(result)); + } + const TableOptions table_options = { + FLAGS_table_name_max_chars, FLAGS_table_inputs_max_chars}; + std::cout << (FLAGS_output_format == "table" + ? format_table(results, table_options) + : FLAGS_output_format == "json" + ? format_condensed_report(results) + : format_report(results)) + << std::endl; + return failed ? 1 : 0; + } catch (const std::exception& e) { + std::cerr << e.what() << std::endl; + return 2; + } +} diff --git a/backends/vulkan/test/ops/benchmark_v2/examples/buffer_shader.json b/backends/vulkan/test/ops/benchmark_v2/examples/buffer_shader.json new file mode 100644 index 00000000000..2e5df2180eb --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/examples/buffer_shader.json @@ -0,0 +1,29 @@ +{ + "shader": "buffer_to_buffer_float", + "gwg_size": "default_pick_gwg", + "lwg_size": "default_pick_lwg", + "target_ms": 10, + "benchmark_runs": 5, + "bindings": [ + { + "buffer": { + "name": "output", + "sizes": [8], + "access": "write", + "data": {"fill": 0} + } + }, + { + "buffer": { + "name": "input", + "sizes": [8], + "data": {"rand_uniform_float": [-1, 1]} + } + }, + { + "meta_ubo": "output", + "field": "numel" + } + ], + "outputs": ["output"] +} diff --git a/backends/vulkan/test/ops/benchmark_v2/examples/conv2d.json b/backends/vulkan/test/ops/benchmark_v2/examples/conv2d.json new file mode 100644 index 00000000000..f05511f47ee --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/examples/conv2d.json @@ -0,0 +1,62 @@ +{ + "name": "conv2d", + "operator": "aten.convolution.default", + "target_ms": 10, + "benchmark_runs": 5, + "show_memory_throughput": true, + "show_compute_throughput": true, + "max_dispatches": 512, + "sweep_variables": { + "v_DTYPE": ["float32", "float16"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 4, 4, 4], + "dtype": "v_DTYPE", + "storage": "texture3d", + "layout": "channels", + "data": {"rand_uniform_float": [-1, 1]} + } + }, + { + "tensor": { + "name": "weight", + "sizes": [8, 4, 3, 3], + "dtype": "v_DTYPE", + "storage": "constant", + "layout": "width", + "data": {"rand_uniform_float": [-1, 1]} + } + }, + { + "tensor": { + "name": "bias", + "sizes": [8], + "dtype": "v_DTYPE", + "storage": "constant", + "layout": "width", + "data": {"rand_uniform_float": [-1, 1]} + } + }, + {"int_list": [1, 1]}, + {"int_list": [1, 1]}, + {"int_list": [1, 1]}, + {"bool": false}, + {"int_list": [0, 0]}, + {"int": 1}, + { + "tensor": { + "name": "output", + "sizes": [1, 8, 4, 4], + "dtype": "v_DTYPE", + "storage": "texture3d", + "layout": "channels", + "data": {"fill": 0} + } + } + ] + ] +} diff --git a/backends/vulkan/test/ops/benchmark_v2/examples/conv2d_routes.json b/backends/vulkan/test/ops/benchmark_v2/examples/conv2d_routes.json new file mode 100644 index 00000000000..8ea0b12ff1b --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/examples/conv2d_routes.json @@ -0,0 +1,110 @@ +{ + "name": "conv2d routes", + "operator": "aten.convolution.default", + "target_ms": 10, + "benchmark_runs": 5, + "show_memory_throughput": true, + "show_compute_throughput": true, + "max_dispatches": 512, + "route_sweep": [ + "auto", + "direct", + "im2col", + { + "name": "im2col", + "options": { + "intermediate_storage": "buffer" + } + }, + { + "name": "im2col", + "options": { + "intermediate_storage": "texture2d" + } + }, + { + "name": "im2col", + "options": { + "intermediate_storage": "texture3d" + } + } + ], + "validation_method": { + "function": "is_close", + "atol": 0.05, + "rtol": 0.01 + }, + "sweep_variables": { + "v_DTYPE": ["float32", "float16"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 32, 32], + "dtype": "v_DTYPE", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 32, 3, 3], + "dtype": "v_DTYPE", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-0.05, 0.05] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "v_DTYPE", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 32, 32], + "dtype": "v_DTYPE", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] +} diff --git a/backends/vulkan/test/ops/benchmark_v2/examples/linear.json b/backends/vulkan/test/ops/benchmark_v2/examples/linear.json new file mode 100644 index 00000000000..abb5f2217e7 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/examples/linear.json @@ -0,0 +1,101 @@ +{ + "name": "linear", + "operator": "aten.linear.default", + "target_ms": 10, + "benchmark_runs": 5, + "show_memory_throughput": true, + "show_compute_throughput": true, + "max_dispatches": 512, + "validation_method": {"function": "is_close", "atol": 0.05, "rtol": 0.01}, + "sweep_variables": { + "v_DTYPE": ["float32", "float16"], + "v_M": [1, 16, 128], + "v_STORAGE": ["buffer", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": ["v_M", 2048], + "dtype": "v_DTYPE", + "storage": "v_STORAGE", + "layout": "width", + "data": {"rand_uniform_float": [-1, 1]} + } + }, + { + "tensor": { + "name": "weight", + "sizes": [2048, 2048], + "dtype": "v_DTYPE", + "storage": "constant", + "layout": "width", + "data": {"rand_uniform_float": [-0.02, 0.02]} + } + }, + { + "tensor": { + "name": "bias", + "sizes": [2048], + "dtype": "v_DTYPE", + "storage": "constant", + "layout": "width", + "data": {"rand_uniform_float": [-1, 1]} + } + }, + { + "tensor": { + "name": "output", + "sizes": ["v_M", 2048], + "dtype": "v_DTYPE", + "storage": "v_STORAGE", + "layout": "width", + "data": {"fill": 0} + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": ["v_M", 2048], + "dtype": "v_DTYPE", + "storage": "v_STORAGE", + "layout": "width", + "data": {"rand_uniform_float": [-1, 1]} + } + }, + { + "tensor": { + "name": "weight", + "sizes": [8192, 2048], + "dtype": "v_DTYPE", + "storage": "constant", + "layout": "width", + "data": {"rand_uniform_float": [-0.02, 0.02]} + } + }, + { + "tensor": { + "name": "bias", + "sizes": [8192], + "dtype": "v_DTYPE", + "storage": "constant", + "layout": "width", + "data": {"rand_uniform_float": [-1, 1]} + } + }, + { + "tensor": { + "name": "output", + "sizes": ["v_M", 8192], + "dtype": "v_DTYPE", + "storage": "v_STORAGE", + "layout": "width", + "data": {"fill": 0} + } + } + ] + ] +} diff --git a/backends/vulkan/test/ops/benchmark_v2/examples/q4gsw_linear.json b/backends/vulkan/test/ops/benchmark_v2/examples/q4gsw_linear.json new file mode 100644 index 00000000000..99928a409ce --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/examples/q4gsw_linear.json @@ -0,0 +1,67 @@ +{ + "name": "q4gsw linear", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 10, + "benchmark_runs": 5, + "show_memory_throughput": true, + "show_compute_throughput": true, + "max_dispatches": 512, + "sweep_variables": { + "v_DTYPE": ["float32", "float16"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4, 32], + "dtype": "v_DTYPE", + "storage": "buffer", + "layout": "width", + "data": {"rand_uniform_float": [-1, 1]} + } + }, + { + "tensor": { + "name": "weight", + "sizes": [16, 16], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": {"rand_uniform_int": [0, 255]} + } + }, + { + "tensor": { + "name": "scales", + "sizes": [1, 16], + "dtype": "v_DTYPE", + "storage": "constant", + "layout": "width", + "data": {"rand_uniform_float": [0.01, 0.1]} + } + }, + {"int": 32}, + { + "tensor": { + "name": "bias", + "sizes": [16], + "dtype": "v_DTYPE", + "storage": "constant", + "layout": "width", + "data": {"rand_uniform_float": [-1, 1]} + } + }, + { + "tensor": { + "name": "output", + "sizes": [4, 16], + "dtype": "v_DTYPE", + "storage": "buffer", + "layout": "width", + "data": {"fill": 0} + } + } + ] + ] +} diff --git a/backends/vulkan/test/ops/benchmark_v2/examples/q8ta_conv2d.json b/backends/vulkan/test/ops/benchmark_v2/examples/q8ta_conv2d.json new file mode 100644 index 00000000000..9670ec2ec62 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/examples/q8ta_conv2d.json @@ -0,0 +1,83 @@ +{ + "name": "q8ta conv2d", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 10, + "benchmark_runs": 5, + "show_memory_throughput": true, + "show_compute_throughput": true, + "max_dispatches": 512, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 4, 4, 4], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": {"rand_uniform_int": [-8, 8]} + } + }, + {"float": 0.25}, + {"int": 0}, + { + "tensor": { + "name": "weight", + "sizes": [8, 36], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": {"rand_uniform_int": [-8, 8]} + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [8], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": {"row_sum": "weight"} + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [8], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": {"rand_uniform_float": [0.01, 0.1]} + } + }, + {"float": 0.25}, + {"int": 0}, + { + "tensor": { + "name": "bias", + "sizes": [8], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": {"rand_uniform_float": [-1, 1]} + } + }, + {"int_list": [3, 3]}, + {"int_list": [1, 1]}, + {"int_list": [1, 1]}, + {"int_list": [1, 1]}, + {"int": 1}, + {"string": "none"}, + { + "tensor": { + "name": "output", + "sizes": [1, 8, 4, 4], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": {"fill": 0} + } + } + ] + ] +} diff --git a/backends/vulkan/test/ops/benchmark_v2/examples/q8ta_conv2d_routes.json b/backends/vulkan/test/ops/benchmark_v2/examples/q8ta_conv2d_routes.json new file mode 100644 index 00000000000..81f7374097b --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/examples/q8ta_conv2d_routes.json @@ -0,0 +1,116 @@ +{ + "name": "q8ta conv2d routes", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 10, + "benchmark_runs": 5, + "show_memory_throughput": true, + "show_compute_throughput": true, + "max_dispatches": 512, + "route_sweep": ["auto", "direct", "im2col"], + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-8, 8] + } + } + }, + { + "float": 0.25 + }, + { + "int": 0 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-8, 8] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.01, 0.1] + } + } + }, + { + "float": 0.25 + }, + { + "int": 0 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] +} diff --git a/backends/vulkan/test/ops/benchmark_v2/examples/shader_workgroup_sweep.json b/backends/vulkan/test/ops/benchmark_v2/examples/shader_workgroup_sweep.json new file mode 100644 index 00000000000..db9f3816498 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/examples/shader_workgroup_sweep.json @@ -0,0 +1,33 @@ +{ + "name": "buffer copy shader and workgroup sweep", + "shader_sweep": ["buffer_to_buffer_float", "buffer_to_buffer_int32"], + "gwg_size": "default_pick_gwg", + "lwg_size_sweep": {"x": [32, 128, 32]}, + "target_ms": 10, + "benchmark_runs": 5, + "show_memory_throughput": true, + "show_compute_throughput": false, + "max_dispatches": 512, + "reference": "copy", + "bindings": [ + { + "buffer": { + "name": "output", + "sizes": [65536], + "dtype": "float32", + "access": "write", + "data": {"fill": 0} + } + }, + { + "buffer": { + "name": "input", + "sizes": [65536], + "dtype": "float32", + "data": {"rand_uniform_float": [-1, 1]} + } + }, + {"meta_ubo": "output", "field": "numel"} + ], + "outputs": ["output"] +} diff --git a/backends/vulkan/test/ops/benchmark_v2/examples/shaders.json b/backends/vulkan/test/ops/benchmark_v2/examples/shaders.json new file mode 100644 index 00000000000..a9551441f4b --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/examples/shaders.json @@ -0,0 +1,79 @@ +[ + { + "name": "buffer copy", + "shader": "buffer_to_buffer_float", + "gwg_size": [8, 1, 1], + "lwg_size": [8, 1, 1], + "target_ms": 10, + "benchmark_runs": 5, + "bindings": [ + { + "buffer": { + "name": "output", + "sizes": [8], + "access": "write", + "data": {"fill": 0} + } + }, + { + "buffer": { + "name": "input", + "sizes": [8], + "data": {"rand_uniform_float": [-1, 1]} + } + }, + { + "meta_ubo": "output", + "field": "numel" + } + ], + "outputs": ["output"] + }, + { + "name": "texture addition", + "shader": "binary_add_texture3d_float", + "gwg_size": "default_pick_gwg", + "lwg_size": "default_pick_lwg", + "target_ms": 10, + "benchmark_runs": 5, + "bindings": [ + { + "texture3d": { + "name": "out", + "sizes": [1, 1, 1, 8], + "access": "write", + "data": {"fill": 0} + } + }, + { + "texture3d": { + "name": "a", + "sizes": [1, 1, 1, 8], + "data": {"rand_uniform_float": [-1, 1]} + } + }, + { + "texture3d": { + "name": "b", + "sizes": [1, 1, 1, 8], + "data": {"rand_uniform_float": [-1, 1]} + } + }, + {"meta_ubo": "out"}, + {"meta_ubo": "a"}, + {"meta_ubo": "b"} + ], + "specialization_constants": [ + {"layout": "out"}, + {"layout": "a"}, + {"layout": "b"} + ], + "push_constants": [ + { + "dtype": "float32", + "data": [2] + } + ], + "outputs": ["out"] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/examples/texture_shader.json b/backends/vulkan/test/ops/benchmark_v2/examples/texture_shader.json new file mode 100644 index 00000000000..6c83fef1c06 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/examples/texture_shader.json @@ -0,0 +1,47 @@ +{ + "name": "texture addition", + "shader": "binary_add_texture3d_float", + "gwg_size": [2, 1, 1], + "lwg_size": [2, 1, 1], + "target_ms": 10, + "benchmark_runs": 5, + "bindings": [ + { + "texture3d": { + "name": "out", + "sizes": [1, 1, 1, 8], + "access": "write", + "data": {"fill": 0} + } + }, + { + "texture3d": { + "name": "a", + "sizes": [1, 1, 1, 8], + "data": {"rand_uniform_float": [-1, 1]} + } + }, + { + "texture3d": { + "name": "b", + "sizes": [1, 1, 1, 8], + "data": {"rand_uniform_float": [-1, 1]} + } + }, + {"meta_ubo": "out"}, + {"meta_ubo": "a"}, + {"meta_ubo": "b"} + ], + "specialization_constants": [ + {"layout": "out"}, + {"layout": "a"}, + {"layout": "b"} + ], + "push_constants": [ + { + "dtype": "float32", + "data": [2] + } + ], + "outputs": ["out"] +} diff --git a/backends/vulkan/test/ops/benchmark_v2/examples/texture_shader_lwg_range.json b/backends/vulkan/test/ops/benchmark_v2/examples/texture_shader_lwg_range.json new file mode 100644 index 00000000000..818161f0c7f --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/examples/texture_shader_lwg_range.json @@ -0,0 +1,75 @@ +{ + "name": "texture addition LWG range", + "shader": "binary_add_texture3d_float", + "gwg_size": "default_pick_gwg", + "lwg_size_sweep": { + "x": [4, 12, 4], + "y": [4, 12, 4], + "z": [1, 1, 1] + }, + "target_ms": 10, + "benchmark_runs": 5, + "max_dispatches": 512, + "validation_method": { + "function": "is_close", + "atol": 0.0001, + "rtol": 0.0001 + }, + "bindings": [ + { + "texture3d": { + "name": "out", + "sizes": [1, 16, 64, 64], + "access": "write", + "data": { + "fill": 0 + } + } + }, + { + "texture3d": { + "name": "a", + "sizes": [1, 16, 64, 64], + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "texture3d": { + "name": "b", + "sizes": [1, 16, 64, 64], + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "meta_ubo": "out" + }, + { + "meta_ubo": "a" + }, + { + "meta_ubo": "b" + } + ], + "specialization_constants": [ + { + "layout": "out" + }, + { + "layout": "a" + }, + { + "layout": "b" + } + ], + "push_constants": [ + { + "dtype": "float32", + "data": [2] + } + ], + "outputs": ["out"] +} diff --git a/backends/vulkan/test/ops/benchmark_v2/examples/texture_shader_lwg_sweep.json b/backends/vulkan/test/ops/benchmark_v2/examples/texture_shader_lwg_sweep.json new file mode 100644 index 00000000000..1a5a2f5782c --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/examples/texture_shader_lwg_sweep.json @@ -0,0 +1,76 @@ +{ + "name": "texture addition LWG list", + "shader": "binary_add_texture3d_float", + "gwg_size": "default_pick_gwg", + "lwg_size": [ + [8, 8, 1], + [10, 10, 1] + ], + "target_ms": 10, + "benchmark_runs": 5, + "show_memory_throughput": true, + "show_compute_throughput": true, + "max_dispatches": 512, + "validation_method": { + "function": "is_close", + "atol": 0.0001, + "rtol": 0.0001 + }, + "bindings": [ + { + "texture3d": { + "name": "out", + "sizes": [1, 16, 64, 64], + "access": "write", + "data": { + "fill": 0 + } + } + }, + { + "texture3d": { + "name": "a", + "sizes": [1, 16, 64, 64], + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "texture3d": { + "name": "b", + "sizes": [1, 16, 64, 64], + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "meta_ubo": "out" + }, + { + "meta_ubo": "a" + }, + { + "meta_ubo": "b" + } + ], + "specialization_constants": [ + { + "layout": "out" + }, + { + "layout": "a" + }, + { + "layout": "b" + } + ], + "push_constants": [ + { + "dtype": "float32", + "data": [2] + } + ], + "outputs": ["out"] +} diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/CaseCheckRegistry.cpp b/backends/vulkan/test/ops/benchmark_v2/framework/CaseCheckRegistry.cpp new file mode 100644 index 00000000000..bdbdffd82cf --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/CaseCheckRegistry.cpp @@ -0,0 +1,66 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include + +namespace vkcompute::shader_profiling { + +void CaseCheckRegistry::register_check( + const std::string& name, + CaseCheckFunction function) { + if (name.empty() || !function) + throw std::invalid_argument("Case check requires a name and callable"); + if (!checks_.emplace(name, function).second) + throw std::invalid_argument("Duplicate case check: " + name); +} + +CaseCheckFunction CaseCheckRegistry::get_check(const std::string& name) const { + const auto entry = checks_.find(name); + if (entry == checks_.end()) + throw std::invalid_argument("Unknown case check: " + name); + return entry->second; +} + +CaseCheckRegistry& case_check_registry() { + static CaseCheckRegistry registry; + return registry; +} + +Report run_case_check(const Json& spec) { + const std::string name = spec.at("check"); + Report result{{"case_name", spec.value("name", name)}, {"check_name", name}}; + try { + GraphConfig config; + config.force_resize = true; + ComputeGraph graph(config); + Json validation = case_check_registry().get_check(name)(spec, graph); + const std::string status = validation.value("status", ""); + if (status != "PASS" && status != "FAIL" && status != "SKIP") + throw std::invalid_argument( + "Case check must return a PASS, FAIL, or SKIP status"); + result["validation"] = Report(validation); + if (status == "SKIP") { + result["status"] = "SKIP"; + result["skip_reason"] = + validation.value("reason", "Case check unsupported"); + } + } catch (const UnsupportedRoute& error) { + result["status"] = "SKIP"; + result["skip_reason"] = error.what(); + result["validation"] = {{"status", "SKIP"}}; + } catch (const vkapi::ShaderNotSupportedError& error) { + result["status"] = "SKIP"; + result["skip_reason"] = error.what(); + result["validation"] = {{"status", "SKIP"}}; + } catch (const std::exception& error) { + result["validation"] = {{"status", "FAIL"}, {"error", error.what()}}; + } + return result; +} + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/CaseCheckRegistry.h b/backends/vulkan/test/ops/benchmark_v2/framework/CaseCheckRegistry.h new file mode 100644 index 00000000000..b24f941a7b0 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/CaseCheckRegistry.h @@ -0,0 +1,36 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include +#include + +#include + +namespace vkcompute::shader_profiling { +using CaseCheckFunction = Json (*)(const Json&, ComputeGraph&); +class CaseCheckRegistry final { + private: + std::unordered_map checks_; + + public: + void register_check(const std::string& name, CaseCheckFunction function); + CaseCheckFunction get_check(const std::string& name) const; +}; +CaseCheckRegistry& case_check_registry(); +Report run_case_check(const Json& spec); +} // namespace vkcompute::shader_profiling + +#define REGISTER_CASE_CHECK(name, function) \ + ::vkcompute::shader_profiling::case_check_registry().register_check( \ + #name, &function) +#define REGISTER_CASE_CHECKS \ + static void register_case_checks(); \ + static const ::vkcompute::shader_profiling::RouteRegisterInit \ + case_checks_reg(®ister_case_checks); \ + static void register_case_checks() diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/Config.cpp b/backends/vulkan/test/ops/benchmark_v2/framework/Config.cpp new file mode 100644 index 00000000000..f967bd50793 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/Config.cpp @@ -0,0 +1,531 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include + +#include +#include +#include +#include +#include + +namespace vkcompute::shader_profiling { + +namespace { + +void require(bool condition, const std::string& message) { + if (!condition) { + throw std::invalid_argument(message); + } +} + +uint32_t positive_integer(const Json& value, const std::string& name) { + require(value.is_number_integer(), name + " must be an integer"); + const double number = value.get(); + require(number > 0 && number <= UINT32_MAX, name + " is out of range"); + return value.get(); +} + +std::string +normalize_tensor(Json& tensors, Json tensor, const std::string& default_name) { + require(tensor.is_object(), "Tensor properties must be an object"); + if (tensor.contains("ref")) { + require(tensor.size() == 1, "Tensor references accept only ref"); + require( + tensor.at("ref").is_string() && + !tensor.at("ref").get().empty(), + "ref must be a nonempty tensor name"); + return tensor.at("ref").get(); + } + require(tensor.contains("sizes"), "Tensor requires sizes or ref"); + const Json name = tensor.value("name", Json(default_name)); + require( + name.is_string() && !name.get().empty(), + "Tensor name must be a nonempty string"); + const std::string tensor_name = name.get(); + require( + !tensors.contains(tensor_name), "Duplicate tensor name: " + tensor_name); + tensor.erase("name"); + tensors[tensor_name] = std::move(tensor); + return tensor_name; +} + +Json normalize_shader_config(const Json& spec) { + require( + !spec.contains("tensors"), + "Declare shader tensor properties in bindings, not tensors"); + require( + spec.contains("bindings") && spec.at("bindings").is_array(), + "bindings must be an array in shader binding order"); + Json normalized = spec; + normalized["tensors"] = Json::object(); + auto& bindings = normalized["bindings"]; + for (size_t i = 0; i < bindings.size(); ++i) { + Json& binding = bindings.at(i); + require(binding.is_object(), "Each binding must be an object"); + int kinds = binding.contains("meta_ubo") + binding.contains("uniform"); + const char* type = nullptr; + for (const char* candidate : {"buffer", "texture2d", "texture3d"}) { + if (binding.contains(candidate)) { + ++kinds; + type = candidate; + } + } + require( + kinds == 1, + "Each binding requires exactly one of buffer, texture2d, texture3d, meta_ubo, uniform"); + if (type == nullptr) { + continue; + } + require( + binding.size() == 1, + "Put tensor properties inside the binding type object"); + Json tensor = binding.at(type); + require(tensor.is_object(), "Tensor binding properties must be an object"); + require( + !tensor.contains("storage"), + "Binding type determines storage; omit storage"); + const Json access = tensor.value("access", Json("read")); + tensor.erase("access"); + if (!tensor.contains("ref")) { + tensor["storage"] = type; + } + const std::string tensor_name = normalize_tensor( + normalized["tensors"], + std::move(tensor), + "binding_" + std::to_string(i)); + binding = {{"tensor", tensor_name}, {"storage", type}, {"access", access}}; + } + return normalized; +} + +Json normalize_operator_config(const Json& spec) { + require( + !spec.contains("tensors"), + "Declare operator tensor properties in args_sweep, not tensors"); + require( + !spec.contains("inputs") && !spec.contains("outputs"), + "Use args_sweep instead of inputs and outputs for operators"); + require( + spec.contains("args") && spec.at("args").is_array() && + !spec.at("args").empty(), + "Operator args must be a nonempty ordered array"); + Json normalized = spec; + normalized["tensors"] = Json::object(); + auto& args = normalized["args"]; + for (size_t i = 0; i < args.size(); ++i) { + Json& arg = args.at(i); + if (!arg.contains("tensor")) { + continue; + } + require(arg.size() == 1, "Each argument requires a single typed value"); + Json tensor = arg.at("tensor"); + require(tensor.is_object(), "Tensor properties must be an object"); + const bool has_access = tensor.contains("access"); + const Json access = tensor.value("access", Json("read")); + tensor.erase("access"); + const std::string name = normalize_tensor( + normalized["tensors"], std::move(tensor), "arg_" + std::to_string(i)); + arg = {{"tensor", name}}; + if (has_access) { + arg["access"] = access; + } + } + return normalized; +} + +void validate_config(const Json& spec) { + require(spec.is_object(), "Each config must be an object"); + require( + spec.contains("shader") != spec.contains("operator"), + "Specify exactly one of shader or operator"); + const bool is_operator = spec.contains("operator"); + const char* kind = is_operator ? "operator" : "shader"; + require( + !spec.contains("global_size") && !spec.contains("local_size"), + "Use gwg_size and lwg_size for workgroup settings"); + require( + spec.at(kind).is_string(), + std::string(kind) + " must be a registry name"); + if (!is_operator) { + for (const char* key : {"gwg_size", "lwg_size"}) { + require(spec.contains(key), std::string("Missing ") + key); + if (spec.at(key).is_string()) { + require( + !spec.at(key).get().empty(), + std::string(key) + " picker name must not be empty"); + continue; + } + require( + spec.contains(key) && spec.at(key).is_array() && + spec.at(key).size() == 3, + std::string(key) + + " must be a picker name or three positive integers"); + for (const Json& dim : spec.at(key)) { + positive_integer(dim, key); + } + } + } + const Json tensors = spec.value("tensors", Json::object()); + require(tensors.is_object(), "tensors must be an object keyed by name"); + for (const auto& item : tensors.items()) { + const Json& tensor = item.value(); + require( + tensor.is_object() && tensor.contains("sizes"), + "Tensor requires sizes"); + tensor_numel(tensor.at("sizes")); + const std::string dtype = tensor.value("dtype", "float32"); + require( + dtype == "float32" || dtype == "float16" || dtype == "int32" || + dtype == "int8" || dtype == "uint8" || dtype == "int8x4", + "Unsupported tensor dtype: " + dtype); + const std::string storage = tensor.value("storage", "buffer"); + require( + storage == "buffer" || storage == "texture2d" || + storage == "texture3d" || (is_operator && storage == "constant"), + "Unsupported tensor storage: " + storage); + if (storage != "buffer" && storage != "constant") { + require( + tensor.at("sizes").size() <= 4, + "Textures support at most four dimensions"); + } + const std::string layout = tensor.value("layout", "width"); + require( + layout == "width" || layout == "height" || layout == "channels" || + layout == "4w" || layout == "4c" || layout == "4w4c" || + layout == "4h4w" || layout == "4c1w", + "Unsupported tensor layout: " + layout); + const bool packed_layout = + layout != "width" && layout != "height" && layout != "channels"; + require( + (dtype == "int8x4") == packed_layout, + "int8x4 requires a packed int8 layout; other dtypes require width, height, or channels"); + if (dtype == "int8x4") { + require( + (storage == "buffer" || storage == "constant") && + tensor.at("sizes").size() <= 4, + "int8x4 requires buffer or constant storage with at most four dimensions"); + } + require( + !tensor.contains("constant"), + "Use storage: constant instead of the tensor constant flag"); + validate_data_initializer(tensor, tensor_numel(tensor.at("sizes"))); + } + if (is_operator) { + for (const char* key : + {"gwg_size", + "lwg_size", + "lwg_size_sweep", + "bindings", + "specialization_constants", + "push_constants"}) { + require( + !spec.contains(key), + std::string(key) + " is only valid for raw shaders"); + } + require( + spec.contains("args") && spec.at("args").is_array(), + "Operator args must be an ordered argument array"); + for (const Json& arg : spec.at("args")) { + if (arg.is_null()) { + continue; + } + require(arg.is_object(), "Each argument must be null or a typed value"); + if (arg.contains("tensor")) { + require( + arg.size() == 1 || (arg.size() == 2 && arg.contains("access")), + "Tensor argument accepts only tensor and optional access"); + const Json& value = arg.at("tensor"); + require( + value.is_string() && tensors.contains(value.get()), + "Unknown argument tensor"); + const Json access = arg.value("access", Json("read")); + require( + access == "read" || access == "write" || access == "read_write", + "Tensor access must be read, write, or read_write"); + if (access != "read") { + require( + tensors.at(value.get()).value("storage", "buffer") != + "constant", + "Writable tensors cannot be constant"); + } + continue; + } + require( + arg.size() == 1, + "Each argument must be null or a single typed value"); + const auto item = arg.items().begin(); + const std::string key = item.key(); + const Json& value = item.value(); + if (key == "int") { + require( + value.is_number_integer() && value.get() >= INT32_MIN && + value.get() <= INT32_MAX, + "int argument must fit int32"); + } else if (key == "float") { + require( + value.is_number() && std::isfinite(value.get()), + "float argument must be finite"); + } else if (key == "bool") { + require(value.is_boolean(), "bool argument must be boolean"); + } else if (key == "string") { + require(value.is_string(), "string argument must be a string"); + } else if (key == "int_list") { + require(value.is_array(), "int_list must be an array"); + for (const Json& number : value) { + require( + number.is_number_integer() && number.get() >= INT32_MIN && + number.get() <= INT32_MAX, + "int_list values must fit int32"); + } + } else { + require(false, "Unsupported operator argument type: " + key); + } + } + } else { + require( + !spec.contains("route") && !spec.contains("route_sweep"), + "route and route_sweep are only valid for operators"); + require( + !spec.contains("args") && !spec.contains("args_sweep") && + !spec.contains("sweep_variables"), + "args_sweep is only valid for operators"); + require( + spec.contains("bindings") && spec.at("bindings").is_array(), + "bindings must be an array in shader binding order"); + for (const Json& binding : spec.at("bindings")) { + require(binding.is_object(), "Each binding must be an object"); + const int kinds = binding.contains("tensor") + + binding.contains("meta_ubo") + binding.contains("uniform"); + require( + kinds == 1, + "Each binding requires exactly one of tensor, meta_ubo, uniform"); + for (const char* key : {"tensor", "meta_ubo"}) { + if (binding.contains(key)) { + require( + binding.at(key).is_string() && + tensors.contains(binding.at(key).get()), + std::string(key) + " references an unknown tensor"); + } + } + if (binding.contains("tensor")) { + require( + tensors.at(binding.at("tensor").get()).at("storage") == + binding.at("storage"), + "Tensor reference binding type does not match the referenced tensor"); + } + const std::string access = binding.value("access", "read"); + require( + access == "read" || access == "write" || access == "read_write", + "Unsupported binding access: " + access); + } + if (spec.contains("specialization_constants")) { + require( + spec.at("specialization_constants").is_array(), + "specialization_constants must be an array"); + for (const Json& constant : spec.at("specialization_constants")) { + require( + constant.is_object(), + "Each specialization constant must be an object"); + if (constant.contains("layout")) { + require( + constant.at("layout").is_string() && + tensors.contains(constant.at("layout").get()), + "layout references an unknown tensor binding"); + } + } + } + if (spec.contains("outputs")) { + require( + spec.at("outputs").is_array(), + "outputs must be an array of tensor binding names"); + for (const Json& name : spec.at("outputs")) { + require( + name.is_string() && tensors.contains(name.get()), + "Unknown output tensor binding"); + } + } + } + if (spec.contains("dump_outputs")) { + require( + spec.at("dump_outputs").is_boolean(), "dump_outputs must be boolean"); + } + for (const char* key : + {"show_memory_throughput", "show_compute_throughput"}) { + if (spec.contains(key)) { + require(spec.at(key).is_boolean(), std::string(key) + " must be boolean"); + } + } + if (spec.contains("reference")) { + require( + spec.at("reference").is_string() && + !spec.at("reference").get().empty(), + "reference must be a nonempty function name"); + } + if (spec.contains("validation_method")) { + const Json& method = spec.at("validation_method"); + require(method.is_object(), "validation_method must be an object"); + require( + method.contains("function") && method.at("function") == "is_close", + "validation_method.function must be is_close"); + for (const auto& option : method.items()) { + require( + option.key() == "function" || option.key() == "atol" || + option.key() == "rtol" || option.key() == "outputs", + "validation_method accepts only function, atol, rtol, outputs"); + if (option.key() == "atol" || option.key() == "rtol") { + require( + option.value().is_number() && + std::isfinite(option.value().get()) && + option.value().get() >= 0, + "Validation tolerance must be finite and nonnegative"); + } + } + validation_output_names(spec); + } + for (const char* key : {"benchmark_runs", "max_dispatches"}) { + if (spec.contains(key)) { + const uint32_t number = positive_integer(spec.at(key), key); + require(number <= 65536, std::string(key) + " must not exceed 65536"); + } + } + if (spec.contains("warmup_runs")) { + require( + spec.at("warmup_runs").is_number_integer() && + spec.at("warmup_runs").get() >= 0 && + spec.at("warmup_runs").get() <= 65536, + "warmup_runs must be in [0, 65536]"); + } + if (spec.contains("target_ms")) { + require( + spec.at("target_ms").is_number() && + spec.at("target_ms").get() > 0 && + spec.at("target_ms").get() <= 10000, + "target_ms must be in (0, 10000]"); + } +} + +} // namespace + +std::set validation_output_names(const Json& spec) { + const Json method = spec.value("validation_method", Json::object()); + std::set names; + if (!method.contains("outputs")) + return names; + const Json& selected = method.at("outputs"); + require( + selected.is_array() && !selected.empty(), + "validation_method.outputs must be a nonempty list"); + for (const Json& name : selected) { + require( + name.is_string() && !name.get().empty() && + spec.at("tensors").contains(name.get()), + "validation_method.outputs references an unknown tensor"); + const std::string value = name; + require( + names.insert(value).second, "Duplicate validation output: " + value); + require( + spec.at("tensors").at(value).value("storage", "buffer") != "constant", + "Validation output must be a GPU tensor: " + value); + } + return names; +} + +std::vector parse_configs(const Json& document) { + std::vector definitions; + if (document.is_array()) { + require(!document.empty(), "Config list must not be empty"); + require( + document.size() <= 65536, "Config list must not exceed 65536 cases"); + definitions = document.get>(); + } else { + definitions.push_back(document); + } + std::vector configs; + for (const Json& definition : definitions) { + require(definition.is_object(), "Each config must be an object"); + require( + !definition.contains("force_narrow_int4_tile"), + "force_narrow_int4_tile is not supported by benchmark_v2"); + for (const char* key : + {"force_resize", + "invocations_per_batch", + "gpu_timing", + "shader_filter", + "requirements", + "device_variables"}) { + require( + !definition.contains(key), + std::string(key) + " is not supported by benchmark_v2"); + } + require( + !definition.contains("implementation") && + !definition.contains("implementation_sweep"), + "Use route and route_sweep for operator route selection"); + require( + definition.contains("operator") + definition.contains("shader") + + definition.contains("shader_sweep") + + definition.contains("check") == + 1, + "Specify exactly one of operator, shader, shader_sweep, or check"); + if (definition.contains("check")) { + require( + definition.at("check").is_string() && + !definition.at("check").get().empty(), + "check must be a nonempty registry name"); + require( + !definition.contains("args_sweep") && !definition.contains("args") && + !definition.contains("bindings") && + !definition.contains("tensors"), + "Standalone checks define their inputs in parameters or operator_case"); + configs.push_back(definition); + continue; + } + const bool is_operator = definition.contains("operator"); + const auto expanded = is_operator ? expand_operator_sweep(definition) + : expand_shader_sweep(definition); + require( + expanded.size() <= 65536 - configs.size(), + "Config list must not exceed 65536 cases"); + for (Json config : expanded) { + if (is_operator) { + config = normalize_operator_config(config); + } else { + config = normalize_shader_config(config); + } + validate_config(config); + configs.push_back(std::move(config)); + } + } + return configs; +} + +size_t tensor_numel(const Json& sizes) { + require( + sizes.is_array() && !sizes.empty() && sizes.size() <= 8, + "sizes must contain one to eight positive integers"); + size_t count = 1; + for (const Json& size : sizes) { + const uint32_t dim = positive_integer(size, "Tensor dimension"); + require(count <= INT32_MAX / dim, "Tensor element count must fit int32"); + count *= dim; + } + return count; +} + +uint32_t +choose_dispatch_count(double probe_us, double target_us, uint32_t limit) { + require( + limit > 0 && std::isfinite(probe_us) && probe_us >= 0 && + std::isfinite(target_us) && target_us > 0, + "Invalid calibration parameters"); + const double count = std::ceil(target_us / std::max(probe_us, 0.001)); + return static_cast( + std::clamp(count, 1.0, static_cast(limit))); +} + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/Config.h b/backends/vulkan/test/ops/benchmark_v2/framework/Config.h new file mode 100644 index 00000000000..99c3cb75e60 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/Config.h @@ -0,0 +1,28 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include + +#include +#include +#include +#include +#include + +namespace vkcompute::shader_profiling { + +using Json = nlohmann::json; + +// Expands sweeps and normalizes inline tensor definitions. +std::vector parse_configs(const Json& document); +std::set validation_output_names(const Json& spec); +size_t tensor_numel(const Json& sizes); +uint32_t +choose_dispatch_count(double probe_us, double target_us, uint32_t limit); + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/DataInitializer.cpp b/backends/vulkan/test/ops/benchmark_v2/framework/DataInitializer.cpp new file mode 100644 index 00000000000..5f068e872f9 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/DataInitializer.cpp @@ -0,0 +1,433 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace vkcompute::shader_profiling { +namespace { +void require(bool condition, const std::string& message) { + if (!condition) + throw std::invalid_argument(message); +} + +template +double numeric_value(const Json& value) { + require(value.is_number(), "Input values must be numbers"); + const double number = value.get(); + require( + std::isfinite(number) && + number >= double(std::numeric_limits::lowest()) && + number <= double(std::numeric_limits::max()), + "Input value is out of dtype range"); + if constexpr (std::is_integral_v) + require( + std::trunc(number) == number, "Integer input must not have a fraction"); + return number; +} +uint32_t random_seed(const Json& value) { + require( + value.is_number_integer() && value.get() >= 0 && + value.get() <= UINT32_MAX, + "seed must fit uint32"); + return value.get(); +} +void integer_bounds(double lower, double upper) { + require( + std::trunc(lower) == lower && std::trunc(upper) == upper, + "Integer random bounds must not have a fraction"); + require( + lower >= INT32_MIN && upper <= UINT32_MAX && upper - lower <= UINT32_MAX, + "Integer random bounds require at most a uint32-sized range"); +} + +template +std::vector +pack_data(const Json& spec, size_t count, uint32_t seed, bool generate) { + require(count <= SIZE_MAX / sizeof(T), "Input byte size overflows"); + const Json data = spec.value("data", Json()); + const Json* values = nullptr; + bool repeat = false, random = false, random_integer = false; + double lower = 0, upper = 0; + uint64_t modulus = 0; + Json fill = spec.value("fill", Json(0)); + if (spec.contains("data")) { + require(!spec.contains("fill"), "Specify either data or fill"); + require( + data.is_array() || data.is_object(), + "data must be an initializer object or array"); + if (data.is_array()) { + require( + data.size() == count, + "data must contain exactly one value per element"); + values = &data; + } else if (data.contains("repeat")) { + require( + data.size() == 1 && data.at("repeat").is_array() && + !data.at("repeat").empty(), + "repeat requires a nonempty array"); + values = &data.at("repeat"); + repeat = true; + } else if ( + data.contains("row_sum") || data.contains("column_sum") || + data.contains("q4_group_sum")) { + require(data.size() == 1, "Derived initializers require a single method"); + if (data.contains("q4_group_sum")) { + const Json& method = data.at("q4_group_sum"); + require( + method.is_object() && method.size() == 2 && + method.contains("tensor") && method.contains("group_size"), + "q4_group_sum requires tensor and group_size"); + require( + method.at("tensor").is_string() && + !method.at("tensor").get().empty(), + "q4_group_sum requires a source tensor name"); + const Json& group = method.at("group_size"); + require( + group.is_number_integer() && group.get() > 0 && + group.get() <= INT32_MAX, + "q4_group_sum group_size must be a positive int32"); + require( + (std::is_same_v || std::is_same_v), + "q4_group_sum requires int32 or floating point output data"); + } else { + const char* method = + data.contains("row_sum") ? "row_sum" : "column_sum"; + require( + data.at(method).is_string() && + !data.at(method).get().empty(), + std::string(method) + " requires a source tensor name"); + require( + (std::is_same_v), + std::string(method) + " requires int32 output data"); + } + require(!generate, "Derived initializers require tensor data context"); + return {}; + } else if (data.contains("fill")) { + require(data.size() == 1, "fill initializer accepts only fill"); + fill = data.at("fill"); + } else if (data.contains("abs_mod")) { + require(data.size() == 1, "abs_mod accepts only its method object"); + const Json& method = data.at("abs_mod"); + require( + method.is_object() && method.contains("bounds") && + method.contains("mod") && + (method.size() == 2 || + (method.size() == 3 && method.contains("seed"))), + "abs_mod requires bounds, mod, and optional seed"); + const Json& bounds = method.at("bounds"); + require( + bounds.is_array() && bounds.size() == 2, + "abs_mod bounds require [min, max]"); + lower = numeric_value(bounds.at(0)); + upper = numeric_value(bounds.at(1)); + const Json& mod = method.at("mod"); + require( + mod.is_number_integer() && mod.get() > 0 && + mod.get() <= UINT32_MAX, + "abs_mod mod must be a positive uint32"); + modulus = mod.get(); + numeric_value(Json(modulus - 1)); + if (method.contains("seed")) + seed = random_seed(method.at("seed")); + random = true; + random_integer = true; + } else { + random_integer = data.contains("rand_uniform_int"); + require( + random_integer || data.contains("rand_uniform_float"), + "Unknown data initializer"); + const char* method = + random_integer ? "rand_uniform_int" : "rand_uniform_float"; + require( + data.size() == 1 || (data.size() == 2 && data.contains("seed")), + "Random initializers accept only bounds and optional seed"); + require( + random_integer || std::is_floating_point_v, + "Floating point random initializer requires floating point dtype"); + const Json& bounds = data.at(method); + require( + bounds.is_array() && bounds.size() == 2, + "Random initializer requires [min, max]"); + lower = numeric_value(bounds.at(0)); + upper = numeric_value(bounds.at(1)); + if (data.contains("seed")) + seed = random_seed(data.at("seed")); + random = true; + } + } + if (values) + for (const Json& value : *values) + numeric_value(value); + else if (!random) + numeric_value(fill); + if (random) { + require(lower <= upper, "Random initializer min must not exceed max"); + if (random_integer) + integer_bounds(lower, upper); + } + if (!generate) + return {}; + std::mt19937 engine(seed); + std::vector bytes(count * sizeof(T)); + for (size_t i = 0; i < count; ++i) { + double number; + if (random_integer) { + const uint64_t span = uint64_t(int64_t(upper) - int64_t(lower)) + 1; + const uint64_t range = uint64_t{1} << 32, limit = range - range % span; + uint32_t draw; + do { + draw = engine(); + } while (draw >= limit); + int64_t value = int64_t(lower) + int64_t(uint64_t(draw) % span); + if (modulus) { + const uint64_t absolute = + value < 0 ? uint64_t(-value) : uint64_t(value); + value = int64_t(absolute % modulus); + } + number = double(value); + } else if (random) { + const double unit = double(engine()) / (double(UINT32_MAX) + 1); + number = lower + (upper - lower) * unit; + } else + number = numeric_value( + values ? values->at(repeat ? i % values->size() : i) : fill); + const T value = static_cast(number); + std::memcpy(bytes.data() + i * sizeof(T), &value, sizeof(T)); + } + return bytes; +} + +std::vector build_data( + const Json& spec, + size_t count, + uint32_t seed, + bool generate, + size_t depth = 0) { + require(depth <= 64, "Data initializer nesting exceeds 64 levels"); + const Json data = spec.value("data", Json()); + if (data.is_object() && data.contains("pad_columns")) { + require(!spec.contains("fill"), "Specify either data or fill"); + require(data.size() == 1, "pad_columns accepts only its method object"); + const Json& method = data.at("pad_columns"); + require( + method.is_object() && method.size() == 2 && method.contains("source") && + method.contains("valid_columns"), + "pad_columns requires source and valid_columns"); + require( + spec.contains("sizes") && tensor_numel(spec.at("sizes")) == count, + "pad_columns requires sizes matching element count"); + const size_t columns = spec.at("sizes").back().get(); + const Json& valid = method.at("valid_columns"); + require( + valid.is_number_integer() && valid.get() >= 0 && + valid.get() <= double(columns), + "pad_columns valid_columns must be between zero and row width"); + const Json& source = method.at("source"); + require( + source.is_array() || source.is_object(), + "pad_columns source must be an initializer"); + require( + !source.is_object() || + (!source.contains("row_sum") && !source.contains("column_sum") && + !source.contains("q4_group_sum")), + "pad_columns source must not require tensor context"); + Json nested = spec; + nested["data"] = source; + auto result = build_data(nested, count, seed, generate, depth + 1); + if (generate) { + const size_t element_size = result.size() / count, + valid_columns = valid.get(); + for (size_t row = 0; row < count / columns; ++row) + std::memset( + result.data() + (row * columns + valid_columns) * element_size, + 0, + (columns - valid_columns) * element_size); + } + return result; + } + const std::string dtype = spec.value("dtype", "float32"); + if (dtype == "float32" || dtype == "float16") + return pack_data(spec, count, seed, generate); + if (dtype == "int32") + return pack_data(spec, count, seed, generate); + if (dtype == "uint32") + return pack_data(spec, count, seed, generate); + if (dtype == "int8" || dtype == "int8x4") + return pack_data(spec, count, seed, generate); + if (dtype == "uint8") + return pack_data(spec, count, seed, generate); + throw std::invalid_argument("Unsupported data dtype: " + dtype); +} + +template +std::vector integer_sums( + const std::vector& source, + size_t segments, + size_t count, + bool columns) { + require( + segments > 0 && count >= segments && + source.size() % (segments * sizeof(T)) == 0, + "Derived sum output is too short or source shape is invalid"); + const size_t contributions = source.size() / (segments * sizeof(T)); + std::vector result(count * sizeof(int32_t)); + for (size_t segment = 0; segment < segments; ++segment) { + int64_t sum = 0; + for (size_t i = 0; i < contributions; ++i) { + const size_t index = + columns ? i * segments + segment : segment * contributions + i; + T value; + std::memcpy(&value, source.data() + index * sizeof(T), sizeof(T)); + sum += value; + } + require( + sum >= INT32_MIN && sum <= INT32_MAX, "Derived sum overflows int32"); + const int32_t value = int32_t(sum); + std::memcpy(result.data() + segment * sizeof(value), &value, sizeof(value)); + } + return result; +} +std::vector sum_integer_tensor( + const Json& spec, + const std::vector& source, + size_t segments, + size_t count, + bool columns) { + const std::string dtype = spec.value("dtype", "float32"); + if (dtype == "int8" || dtype == "int8x4") + return integer_sums(source, segments, count, columns); + if (dtype == "uint8") + return integer_sums(source, segments, count, columns); + if (dtype == "int32") + return integer_sums(source, segments, count, columns); + if (dtype == "uint32") + return integer_sums(source, segments, count, columns); + throw std::invalid_argument("Derived sum source must have integer dtype"); +} +std::vector q4_group_sums( + const Json& spec, + const Json& source_spec, + const std::vector& source) { + const std::string source_dtype = source_spec.value("dtype", "float32"); + require( + source_dtype == "int8" || source_dtype == "uint8", + "q4_group_sum source must contain packed int8/uint8 weights"); + const Json& input_sizes = source_spec.at("sizes"); + const Json& output_sizes = spec.at("sizes"); + require( + input_sizes.size() == 2 && output_sizes.size() == 2, + "q4_group_sum requires [N,K/2] weights and [K/group_size,N] sums"); + const size_t channels = input_sizes.at(0).get(), + packed_width = input_sizes.at(1).get(), + width = packed_width * 2; + const size_t group_size = + spec.at("data").at("q4_group_sum").at("group_size").get(); + require( + width % group_size == 0, + "q4_group_sum group_size must divide unpacked weight width"); + const size_t groups = width / group_size, + output_channels = output_sizes.at(1).get(); + require( + output_sizes.at(0).get() == groups && output_channels >= channels, + "q4_group_sum output shape does not match groups and channels"); + require( + source.size() == channels * packed_width, + "q4_group_sum source byte size does not match weights"); + const bool floating = spec.value("dtype", "float32") != "int32"; + std::vector result(groups * output_channels * sizeof(int32_t)); + for (size_t group = 0; group < groups; ++group) + for (size_t channel = 0; channel < channels; ++channel) { + int64_t sum = 0; + for (size_t k = group * group_size; k < (group + 1) * group_size; ++k) { + const uint8_t value = source.at(channel * packed_width + k / 2); + sum += int((value >> ((k % 2) * 4)) & 15) - 8; + } + const size_t index = group * output_channels + channel; + if (floating) { + const float value = float(sum); + std::memcpy( + result.data() + index * sizeof(value), &value, sizeof(value)); + } else { + require( + sum >= INT32_MIN && sum <= INT32_MAX, + "q4_group_sum overflows int32"); + const int32_t value = int32_t(sum); + std::memcpy( + result.data() + index * sizeof(value), &value, sizeof(value)); + } + } + return result; +} +} // namespace + +void validate_data_initializer(const Json& spec, size_t count) { + build_data(spec, count, 0, false); +} +std::vector make_data(const Json& spec, size_t count, uint32_t seed) { + return build_data(spec, count, seed, true); +} +std::map> make_tensor_data( + const Json& tensors) { + std::map> result; + std::set active; + std::function initialize = + [&](const std::string& name) { + if (result.count(name)) + return; + require(tensors.contains(name), "Unknown data source tensor: " + name); + require( + active.insert(name).second, + "Cyclic data initializer reference: " + name); + const Json& spec = tensors.at(name); + const size_t count = tensor_numel(spec.at("sizes")); + validate_data_initializer(spec, count); + const Json data = spec.value("data", Json()); + if (data.is_object() && + (data.contains("row_sum") || data.contains("column_sum") || + data.contains("q4_group_sum"))) { + const bool q4 = data.contains("q4_group_sum"), + columns = data.contains("column_sum"); + const std::string source = q4 + ? data.at("q4_group_sum").at("tensor").get() + : data.at(columns ? "column_sum" : "row_sum").get(); + initialize(source); + const Json& source_spec = tensors.at(source); + if (q4) + result[name] = q4_group_sums(spec, source_spec, result.at(source)); + else { + const size_t segments = + source_spec.at("sizes") + .at(columns ? source_spec.at("sizes").size() - 1 : 0) + .get(); + result[name] = sum_integer_tensor( + source_spec, result.at(source), segments, count, columns); + } + } else { + // FNV-1a preserves per-tensor seeds across platforms and sweep + // entries. + uint32_t seed = 2166136261u; + for (unsigned char ch : name) + seed = (seed ^ ch) * 16777619u; + result[name] = make_data(spec, count, seed); + } + active.erase(name); + }; + for (const auto& tensor : tensors.items()) + initialize(tensor.key()); + return result; +} +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/DataInitializer.h b/backends/vulkan/test/ops/benchmark_v2/framework/DataInitializer.h new file mode 100644 index 00000000000..fadcd995431 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/DataInitializer.h @@ -0,0 +1,25 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include + +#include +#include +#include +#include +#include + +namespace vkcompute::shader_profiling { + +void validate_data_initializer(const nlohmann::json& spec, size_t count); +std::vector +make_data(const nlohmann::json& spec, size_t count, uint32_t seed = 0); +std::map> make_tensor_data( + const nlohmann::json& tensors); + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/GpuClock.cpp b/backends/vulkan/test/ops/benchmark_v2/framework/GpuClock.cpp new file mode 100644 index 00000000000..74f8152abb1 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/GpuClock.cpp @@ -0,0 +1,78 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include +#include +#include +#include + +namespace vkcompute::shader_profiling { + +GpuClock::GpuClock(const std::string& sysfs_root) { + const std::filesystem::path root(sysfs_root); + std::vector> sources{ + {root / "class/kgsl/kgsl-3d0/gpuclk", 1e6}}; + std::error_code error; + auto devices = + std::filesystem::directory_iterator(root / "devices/platform", error); + for (; !error && devices != std::filesystem::directory_iterator(); + devices.increment(error)) { + const auto path = devices->path(); + if (path.extension() == ".mali" || path.filename() == "mali") { + // Pixel Mali cur_freq uses kHz; legacy Mali clock uses MHz. + sources.emplace_back(path / "cur_freq", 1000); + sources.emplace_back(path / "clock", 1); + } + } + std::optional kernel_units; + error.clear(); + devices = std::filesystem::directory_iterator(root / "class/devfreq", error); + for (; !error && devices != std::filesystem::directory_iterator(); + devices.increment(error)) { + const auto path = devices->path(); + const auto name = path.filename().string(); + const auto extension = path.extension(); + const bool kgsl = name.find("kgsl-3d0") != std::string::npos; + const bool sgpu = extension == ".sgpu" || name == "sgpu"; + if (kgsl || sgpu || extension == ".gpu" || extension == ".mali" || + name == "gpu" || name == "mali" || name == "pvrsrvkm") { + // Samsung SGPU devfreq uses kHz; standard devfreq and KGSL use Hz. + sources.emplace_back(path / "cur_freq", sgpu ? 1000 : 1e6); + if (kgsl) + kernel_units = 1; + else if (sgpu) + kernel_units = 1000; + } + } + if (kernel_units) { + sources.emplace_back(root / "kernel/gpu/gpu_clock", *kernel_units); + } + for (const auto& source : sources) { + path_ = source.first.string(); + units_per_mhz_ = source.second; + if (read_mhz()) + return; + } + path_.clear(); +} + +std::optional GpuClock::read_mhz() const { + if (path_.empty()) + return std::nullopt; + std::ifstream input(path_); + double frequency; + if (!(input >> frequency) || !std::isfinite(frequency) || frequency < 0) + return std::nullopt; + input >> std::ws; + if (!input.eof()) + return std::nullopt; + return frequency / units_per_mhz_; +} + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/GpuClock.h b/backends/vulkan/test/ops/benchmark_v2/framework/GpuClock.h new file mode 100644 index 00000000000..7dcfbcde3a7 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/GpuClock.h @@ -0,0 +1,23 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include + +namespace vkcompute::shader_profiling { + +class GpuClock final { + std::string path_; + double units_per_mhz_ = 1; + + public: + explicit GpuClock(const std::string& sysfs_root = "/sys"); + std::optional read_mhz() const; +}; + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/OperatorBenchmark.cpp b/backends/vulkan/test/ops/benchmark_v2/framework/OperatorBenchmark.cpp new file mode 100644 index 00000000000..3dc72a2cd58 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/OperatorBenchmark.cpp @@ -0,0 +1,161 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace vkcompute::shader_profiling { + +namespace { + +Json tensor_accesses( + ComputeGraph& graph, + size_t first, + size_t end, + const std::map& tensors, + ValueRef primary_output) { + std::map first_access; + std::map all_access; + for (size_t i = first; i < end; ++i) { + std::map node_access; + for (const auto& group : graph.execute_nodes()[i]->args()) { + for (ValueRef ref : group.refs) { + node_access[ref] |= group.access; + } + } + for (const auto& entry : node_access) { + first_access.emplace(entry); + all_access[entry.first] |= entry.second; + } + } + Json result = Json::object(); + for (const auto& tensor : tensors) { + const ValueRef ref = tensor.second; + // Mutable input caches can be written before their unchanged prefix is + // read. + const bool reads_existing = (first_access[ref] & vkapi::kRead) || + (ref != primary_output && (all_access[ref] & vkapi::kRead)); + result[tensor.first] = !(all_access[ref] & vkapi::kWrite) ? "read" + : reads_existing ? "read_write" + : "write"; + } + return result; +} + +ValueRef operator_arg( + const Json& arg, + ComputeGraph& graph, + const std::map& tensors) { + if (arg.is_null()) { + return graph.add_none(); + } + if (arg.contains("tensor")) { + return tensors.at(arg.at("tensor").get()); + } + if (arg.contains("int")) { + return graph.add_scalar(arg.at("int").get()); + } + if (arg.contains("float")) { + return graph.add_scalar(arg.at("float").get()); + } + if (arg.contains("bool")) { + return graph.add_scalar(arg.at("bool").get()); + } + if (arg.contains("int_list")) { + return graph.add_scalar_list( + arg.at("int_list").get>()); + } + return graph.add_string(arg.at("string").get()); +} + +} // namespace + +std::vector build_operator_args( + const Json& spec, + ComputeGraph& graph, + const std::map& tensors) { + std::vector args; + for (const Json& arg : spec.at("args")) + args.push_back(operator_arg(arg, graph, tensors)); + return args; +} + +Report benchmark_operator( + const Json& spec, + const TensorData& host_data, + bool validation_only) { + GraphConfig config; + config.force_resize = true; + if (!validation_only) { + config.descriptor_pool_safety_factor *= spec.value("max_dispatches", 200u); + } + ComputeGraph graph(config); + const auto tensors = create_tensors(spec, graph, host_data, validation_only); + const auto args = build_operator_args(spec, graph, tensors); + const size_t first = graph.execute_nodes().size(); + const Json route = build_operator_route(spec, graph, args); + const size_t end = graph.execute_nodes().size(); + if (first == end) { + throw std::invalid_argument("Operator produced no execute nodes"); + } + const Json accesses = + tensor_accesses(graph, first, end, tensors, args.back()); + std::map outputs; + if (validation_only) { + outputs = add_validation_outputs(spec, graph, tensors, accesses); + } + graph.prepare(); + graph.prepack(); + graph.propagate_resize(); + if (validation_only) { + copy_validation_inputs(spec, graph, tensors, host_data); + graph.execute(); + Report result = + operator_report(spec, actual_tensor_dtypes(graph, tensors), accesses); + if (spec.contains("route")) + result["route"] = route; + result["validation"] = + validate_outputs(spec, graph, tensors, outputs, host_data); + return result; + } + auto encode_batch = [&](uint32_t count) { + for (uint32_t i = 0; i < count; ++i) { + for (size_t j = first; j < end; ++j) { + graph.execute_nodes()[j]->encode(&graph); + } + } + }; + // Warm once to establish the layout state used at each replay boundary. + auto& context = *graph.context(); + context.set_cmd(); + encode_batch(1); + vkapi::VulkanFence fence(context.device()); + context.submit_cmd_to_gpu(fence.get_submit_handle(), true); + fence.wait(); + Report result = + operator_report(spec, actual_tensor_dtypes(graph, tensors), accesses); + if (spec.contains("route")) { + result["route"] = route; + } + result["execute_nodes_per_invocation"] = end - first; + result["execute_nodes"] = Report::array(); + for (size_t i = first; i < end; ++i) { + result["execute_nodes"].push_back(graph.execute_nodes()[i]->name()); + } + result.update(measure(spec, graph, encode_batch, Json(result))); + return result; +} + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/OperatorBenchmark.h b/backends/vulkan/test/ops/benchmark_v2/framework/OperatorBenchmark.h new file mode 100644 index 00000000000..d55357c0233 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/OperatorBenchmark.h @@ -0,0 +1,26 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include +#include +#include + +namespace vkcompute::shader_profiling { + +std::vector build_operator_args( + const Json& spec, + ComputeGraph& graph, + const std::map& tensors); + +Report benchmark_operator( + const Json& spec, + const TensorData& host_data, + bool validation_only = false); + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/Reference.cpp b/backends/vulkan/test/ops/benchmark_v2/framework/Reference.cpp new file mode 100644 index 00000000000..beaffcab269 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/Reference.cpp @@ -0,0 +1,307 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace vkcompute::shader_profiling { + +namespace reference { + +void require(bool condition, const std::string& message) { + if (!condition) { + throw std::invalid_argument(message); + } +} + +void parallel_for( + size_t count, + size_t threads, + const std::function& function) { + if (threads == 0) { + threads = std::max(1u, std::thread::hardware_concurrency()); + } + threads = std::min({threads, size_t(8), count}); + std::vector workers; + for (size_t worker = 1; worker < threads; ++worker) { + workers.emplace_back([&, worker] { + for (size_t i = count * worker / threads; + i < count * (worker + 1) / threads; + ++i) { + function(i); + } + }); + } + for (size_t i = 0; threads > 0 && i < count / threads; ++i) { + function(i); + } + for (auto& worker : workers) { + worker.join(); + } +} + +const Tensor& +tensor_arg(const Json& spec, size_t index, const TensorMap& tensors) { + return tensors.at(spec.at("args").at(index).at("tensor").get()); +} + +} // namespace reference + +namespace { + +using Half = executorch::runtime::etensor::Half; + +std::string tensor_dtype( + const Json& spec, + const Json& actual_dtypes, + const std::string& name) { + return actual_dtypes.contains(name) + ? actual_dtypes.at(name).get() + : spec.at("tensors").at(name).value("dtype", "float32"); +} + +template +double read_element(const std::vector& bytes, size_t index) { + T value; + std::memcpy(&value, bytes.data() + index * sizeof(T), sizeof(T)); + return double(value); +} + +reference::Tensor decode_tensor( + const Json& description, + const std::string& actual_dtype, + const std::vector& bytes) { + const std::string source_dtype = description.value("dtype", "float32"); + const bool constant_half = source_dtype == "float16" && + description.value("storage", "buffer") == "constant"; + const size_t element_size = constant_half ? sizeof(Half) + : source_dtype == "float32" || source_dtype == "float16" ? sizeof(float) + : source_dtype == "int32" ? sizeof(int32_t) + : sizeof(uint8_t); + const size_t count = tensor_numel(description.at("sizes")); + reference::require( + bytes.size() == count * element_size, + "Reference tensor byte count does not match its shape and dtype"); + reference::Tensor result{ + description.at("sizes").get>(), + std::vector(count)}; + for (size_t i = 0; i < count; ++i) { + double value; + if (constant_half) { + value = read_element(bytes, i); + } else if (source_dtype == "float32" || source_dtype == "float16") { + value = read_element(bytes, i); + } else if (source_dtype == "int32") { + value = read_element(bytes, i); + } else if (source_dtype == "uint8") { + value = read_element(bytes, i); + } else { + value = read_element(bytes, i); + } + result.values[i] = + actual_dtype == "float16" ? float(Half(float(value))) : value; + } + return result; +} + +std::string output_name(const Json& spec) { + if (spec.contains("operator")) { + return spec.at("args").back().at("tensor"); + } + for (const Json& binding : spec.at("bindings")) { + if (binding.contains("tensor")) { + return binding.at("tensor"); + } + } + throw std::invalid_argument("Reference requires an output tensor binding"); +} + +} // namespace + +std::string reference_name(const Json& spec) { + const auto& registry = reference_registry(); + if (spec.contains("reference")) { + const std::string name = spec.at("reference"); + reference::require( + registry.has_reference(name), "Unknown reference function: " + name); + return name; + } + return spec.contains("operator") + ? registry.operator_reference(spec.at("operator").get()) + : registry.shader_reference(spec.at("shader").get()); +} + +ReferenceOutputs reference_outputs( + const Json& spec, + const Json& actual_dtypes, + const std::map>& host_data, + size_t threads) { + const std::string function = reference_name(spec); + reference::TensorMap tensors; + for (const auto& item : host_data) { + const Json& description = spec.at("tensors").at(item.first); + tensors.emplace( + item.first, + decode_tensor( + description, + tensor_dtype(spec, actual_dtypes, item.first), + item.second)); + } + ReferenceOutputs outputs; + if (const auto callback = + reference_registry().get_reference_outputs(function)) { + outputs = callback(spec, tensors, threads); + } else { + reference::Tensor output = + reference_registry().get_reference(function)(spec, tensors, threads); + const std::string name = output_name(spec); + reference::require( + output.sizes == + spec.at("tensors").at(name).at("sizes").get>(), + "Reference output shape does not match declared output tensor"); + outputs.emplace(name, std::move(output.values)); + } + for (auto& output : outputs) { + reference::require( + spec.at("tensors").contains(output.first), + "Reference produced unknown output tensor: " + output.first); + reference::require( + output.second.size() == + tensor_numel(spec.at("tensors").at(output.first).at("sizes")), + "Reference output element count does not match declared shape: " + + output.first); + if (tensor_dtype(spec, actual_dtypes, output.first) == "float16") { + for (double& value : output.second) + value = float(Half(float(value))); + } + } + return outputs; +} + +Report compare_outputs( + const Json& spec, + const ReferenceOutputs& expected, + const ReferenceOutputs& actual, + const Json& actual_dtypes) { + const Json method = spec.value("validation_method", Json::object()); + reference::require( + method.value("function", "is_close") == "is_close", + "Unknown validation method"); + Report result = { + {"status", "PASS"}, + {"reference", reference_name(spec)}, + {"method", "is_close"}, + {"outputs", Report::object()}}; + const auto selected = validation_output_names(spec); + if (!selected.empty()) { + result["output_selection"] = selected; + for (const auto& name : selected) { + if (!expected.count(name)) { + result["status"] = "FAIL"; + result["outputs"][name] = { + {"status", "FAIL"}, {"error", "Missing reference output"}}; + } + } + } + size_t total_mismatches = 0; + if (expected.empty()) { + result["status"] = "FAIL"; + result["error"] = "Reference produced no outputs"; + } + for (const auto& item : expected) { + if (!selected.empty() && !selected.count(item.first)) + continue; + const std::string dtype = tensor_dtype(spec, actual_dtypes, item.first); + const double default_tolerance = dtype == "float16" ? 1e-2 + : dtype == "float32" ? 1e-4 + : 0; + const double atol = method.value("atol", default_tolerance); + const double rtol = method.value("rtol", default_tolerance); + reference::require( + std::isfinite(atol) && atol >= 0 && std::isfinite(rtol) && rtol >= 0, + "Validation tolerances must be finite and nonnegative"); + Report output = { + {"status", "PASS"}, + {"element_count", item.second.size()}, + {"atol", atol}, + {"rtol", rtol}}; + const auto computed = actual.find(item.first); + if (computed == actual.end() || + computed->second.size() != item.second.size() || + item.second.size() != + tensor_numel(spec.at("tensors").at(item.first).at("sizes"))) { + output["status"] = "FAIL"; + output["error"] = computed == actual.end() + ? "Missing output" + : "Output element count does not match declared shape"; + result["status"] = "FAIL"; + result["outputs"][item.first] = std::move(output); + continue; + } + size_t mismatches = 0; + size_t nonfinite = 0; + double max_abs_error = 0; + double max_rel_error = 0; + Report mismatch_indices = Report::array(); + for (size_t i = 0; i < item.second.size(); ++i) { + const double reference_value = item.second[i]; + const double actual_value = computed->second[i]; + const bool finite = + std::isfinite(reference_value) && std::isfinite(actual_value); + const double error = std::abs(actual_value - reference_value); + if (finite) { + max_abs_error = std::max(max_abs_error, error); + const double relative = reference_value != 0 + ? error / std::abs(reference_value) + : error == 0 ? 0 + : std::numeric_limits::infinity(); + max_rel_error = std::max(max_rel_error, relative); + } else { + ++nonfinite; + } + if (!finite || error > atol + rtol * std::abs(reference_value)) { + ++mismatches; + if (mismatch_indices.size() < 8) { + mismatch_indices.push_back(i); + } + } + } + output["status"] = mismatches == 0 ? "PASS" : "FAIL"; + output["mismatch_count"] = mismatches; + output["nonfinite_count"] = nonfinite; + output["max_abs_error"] = max_abs_error; + output["max_rel_error"] = + std::isfinite(max_rel_error) ? Report(max_rel_error) : Report(); + output["first_mismatch_indices"] = std::move(mismatch_indices); + total_mismatches += mismatches; + if (mismatches > 0) { + result["status"] = "FAIL"; + } + result["outputs"][item.first] = std::move(output); + } + for (const auto& item : actual) { + if (!selected.empty() && !selected.count(item.first)) + continue; + if (!expected.count(item.first)) { + result["status"] = "FAIL"; + result["outputs"][item.first] = { + {"status", "FAIL"}, {"error", "Unexpected output"}}; + } + } + result["mismatch_count"] = total_mismatches; + return result; +} + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/Reference.h b/backends/vulkan/test/ops/benchmark_v2/framework/Reference.h new file mode 100644 index 00000000000..8aa90c4832e --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/Reference.h @@ -0,0 +1,51 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include + +#include +#include +#include +#include + +namespace vkcompute::shader_profiling { + +using ReferenceOutputs = std::map>; + +namespace reference { + +struct Tensor { + std::vector sizes; + std::vector values; +}; + +using TensorMap = std::map; + +void parallel_for( + size_t count, + size_t threads, + const std::function& function); +const Tensor& +tensor_arg(const Json& spec, size_t index, const TensorMap& tensors); +void require(bool condition, const std::string& message); + +} // namespace reference + +std::string reference_name(const Json& spec); +ReferenceOutputs reference_outputs( + const Json& spec, + const Json& actual_dtypes, + const std::map>& host_data, + size_t threads = 0); +Report compare_outputs( + const Json& spec, + const ReferenceOutputs& expected, + const ReferenceOutputs& actual, + const Json& actual_dtypes); + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/ReferenceRegistry.cpp b/backends/vulkan/test/ops/benchmark_v2/framework/ReferenceRegistry.cpp new file mode 100644 index 00000000000..77219dc5206 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/ReferenceRegistry.cpp @@ -0,0 +1,112 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +namespace vkcompute::shader_profiling { + +namespace { + +void register_association( + std::unordered_map& table, + const std::string& name, + const std::string& reference_name, + const std::string& kind) { + reference::require( + !name.empty() && !reference_name.empty(), + kind + " and reference names must be nonempty"); + reference::require( + table.emplace(name, reference_name).second, + "Duplicate " + kind + " reference association: " + name); +} + +const std::string& lookup_association( + const std::unordered_map& table, + const std::string& name, + const std::string& kind) { + const auto entry = table.find(name); + reference::require( + entry != table.end(), + "No correctness reference registered for " + kind + " " + name); + return entry->second; +} + +} // namespace + +ReferenceFunction ReferenceRegistry::get_reference( + const std::string& name) const { + const auto entry = functions_.find(name); + reference::require( + entry != functions_.end(), "Unknown reference function: " + name); + return entry->second; +} + +ReferenceOutputsFunction ReferenceRegistry::get_reference_outputs( + const std::string& name) const { + const auto entry = output_functions_.find(name); + return entry == output_functions_.end() ? nullptr : entry->second; +} + +bool ReferenceRegistry::has_reference(const std::string& name) const { + return functions_.count(name) || output_functions_.count(name); +} + +const std::string& ReferenceRegistry::operator_reference( + const std::string& name) const { + const auto& reference = lookup_association(operators_, name, "operator"); + reference::require( + has_reference(reference), "Unknown reference function: " + reference); + return reference; +} + +const std::string& ReferenceRegistry::shader_reference( + const std::string& name) const { + const auto& reference = lookup_association(shaders_, name, "shader"); + reference::require( + has_reference(reference), "Unknown reference function: " + reference); + return reference; +} + +void ReferenceRegistry::register_reference( + const std::string& name, + ReferenceFunction function) { + reference::require(!name.empty(), "Reference name must be nonempty"); + reference::require( + function != nullptr, "Reference function must not be null"); + reference::require( + !has_reference(name), "Duplicate reference function: " + name); + functions_.emplace(name, function); +} + +void ReferenceRegistry::register_reference_outputs( + const std::string& name, + ReferenceOutputsFunction function) { + reference::require(!name.empty(), "Reference name must be nonempty"); + reference::require( + function != nullptr, "Reference function must not be null"); + reference::require( + !has_reference(name), "Duplicate reference function: " + name); + output_functions_.emplace(name, function); +} + +void ReferenceRegistry::register_operator_reference( + const std::string& name, + const std::string& reference) { + register_association(operators_, name, reference, "operator"); +} + +void ReferenceRegistry::register_shader_reference( + const std::string& name, + const std::string& reference) { + register_association(shaders_, name, reference, "shader"); +} + +ReferenceRegistry& reference_registry() { + static ReferenceRegistry registry; + return registry; +} + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/ReferenceRegistry.h b/backends/vulkan/test/ops/benchmark_v2/framework/ReferenceRegistry.h new file mode 100644 index 00000000000..d32f90e886f --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/ReferenceRegistry.h @@ -0,0 +1,73 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include + +#include + +namespace vkcompute::shader_profiling { + +using ReferenceFunction = + reference::Tensor (*)(const Json&, const reference::TensorMap&, size_t); + +using ReferenceOutputsFunction = + ReferenceOutputs (*)(const Json&, const reference::TensorMap&, size_t); + +class ReferenceRegistry final { + private: + std::unordered_map functions_; + std::unordered_map output_functions_; + std::unordered_map operators_; + std::unordered_map shaders_; + + public: + ReferenceFunction get_reference(const std::string& name) const; + ReferenceOutputsFunction get_reference_outputs(const std::string& name) const; + bool has_reference(const std::string& name) const; + const std::string& operator_reference(const std::string& name) const; + const std::string& shader_reference(const std::string& name) const; + void register_reference(const std::string& name, ReferenceFunction function); + void register_reference_outputs( + const std::string& name, + ReferenceOutputsFunction function); + void register_operator_reference( + const std::string& name, + const std::string& reference); + void register_shader_reference( + const std::string& name, + const std::string& reference); +}; + +class ReferenceRegisterInit final { + public: + explicit ReferenceRegisterInit(void (*initialize)()) { + initialize(); + } +}; + +ReferenceRegistry& reference_registry(); + +} // namespace vkcompute::shader_profiling + +#define REGISTER_REFERENCE_FN(name, function) \ + ::vkcompute::shader_profiling::reference_registry().register_reference( \ + #name, &function) +#define REGISTER_OPERATOR_REFERENCE(name, reference) \ + ::vkcompute::shader_profiling::reference_registry() \ + .register_operator_reference(#name, #reference) +#define REGISTER_SHADER_REFERENCE(name, reference) \ + ::vkcompute::shader_profiling::reference_registry() \ + .register_shader_reference(#name, #reference) +#define REGISTER_REFERENCE_OUTPUTS_FN(name, function) \ + ::vkcompute::shader_profiling::reference_registry() \ + .register_reference_outputs(#name, &function) +#define REGISTER_REFERENCE_FUNCTIONS \ + static void register_references(); \ + static const ::vkcompute::shader_profiling::ReferenceRegisterInit \ + reference_reg(®ister_references); \ + static void register_references() diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/Report.cpp b/backends/vulkan/test/ops/benchmark_v2/framework/Report.cpp new file mode 100644 index 00000000000..58fbdb75317 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/Report.cpp @@ -0,0 +1,823 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace vkcompute::shader_profiling { +namespace { + +void write_report_value( + std::ostream& output, + const Report& value, + size_t depth, + std::string_view field = {}, + bool inline_array = false) { + if ((!value.is_object() && !value.is_array()) || value.empty()) { + output << value.dump(); + return; + } + if (value.is_array() && inline_array) { + output << '['; + for (size_t i = 0; i < value.size(); ++i) { + if (i > 0) { + output << ", "; + } + output << value.at(i).dump(); + } + output << ']'; + return; + } + + const bool object = value.is_object(); + output << (object ? '{' : '[') << '\n'; + for (auto it = value.begin(); it != value.end(); ++it) { + if (it != value.begin()) { + output << ",\n"; + } + output << std::string((depth + 1) * 2, ' '); + const std::string key = object ? it.key() : ""; + if (object) { + output << Report(key).dump() << ": "; + } + constexpr std::string_view clock_suffix = "_observed_gpu_clock_freq_MHz"; + const bool clock_pairs = field.size() >= clock_suffix.size() && + field.substr(field.size() - clock_suffix.size()) == clock_suffix; + const bool compact = key == "sizes" || clock_pairs || + (key == "value" && (field == "gwg_size" || field == "lwg_size")); + write_report_value(output, it.value(), depth + 1, key, compact); + } + output << '\n' << std::string(depth * 2, ' ') << (object ? '}' : ']'); +} + +double percentile(const std::vector& sorted, double fraction) { + const double position = fraction * (sorted.size() - 1); + const size_t lower = static_cast(position); + const size_t upper = std::min(lower + 1, sorted.size() - 1); + return sorted[lower] + (sorted[upper] - sorted[lower]) * (position - lower); +} + +Report case_header(const Json& spec) { + const bool op = spec.contains("operator"); + const std::string name = spec.at(op ? "operator" : "shader"); + Report report = { + {"case_name", spec.value("name", name)}, + {op ? "operator_name" : "shader_name", name}}; + if (spec.contains("sweep_index")) { + report["sweep_index"] = spec.at("sweep_index"); + } + if (op && spec.contains("sweep_variables")) { + report["args_sweep_index"] = spec.at("args_sweep_index"); + report["sweep_variables"] = spec.at("sweep_variables"); + } + if (op && spec.contains("route")) { + report["route"] = {{"requested", spec.at("route")}, {"resolved", nullptr}}; + } + return report; +} + +Report tensor_description(const Json& tensor, const std::string& actual_dtype) { + const std::string requested = tensor.value("dtype", "float32"); + Report info = { + {"type", tensor.value("storage", "buffer")}, + {"sizes", tensor.at("sizes")}, + {"dtype", actual_dtype}, + {"layout", tensor.value("layout", "width")}}; + if (actual_dtype != requested) { + info["requested_dtype"] = requested; + } + return info; +} + +Report describe_bindings(const Json& spec, const Json& actual_dtypes) { + Report result = Report::object(); + const Json& tensors = spec.at("tensors"); + for (size_t i = 0; i < spec.at("bindings").size(); ++i) { + const Json& binding = spec.at("bindings").at(i); + Report info; + std::string name; + if (binding.contains("tensor")) { + name = binding.at("tensor"); + const Json& tensor = tensors.at(name); + info = tensor_description( + tensor, actual_dtypes.value(name, tensor.value("dtype", "float32"))); + info["access"] = binding.value("access", "read"); + if (result.contains(name)) { + info["ref"] = name; + } + } else if (binding.contains("meta_ubo")) { + const std::string tensor = binding.at("meta_ubo"); + const std::string field = binding.value("field", "meta"); + name = tensor + "." + field; + info = {{"type", "meta_ubo"}, {"tensor", tensor}, {"field", field}}; + } else { + name = "binding_" + std::to_string(i); + info = {{"type", "uniform"}, {"values", binding.at("uniform")}}; + } + info["binding_index"] = i; + std::string key = name; + for (size_t suffix = i; result.contains(key); ++suffix) { + key = name + "_" + std::to_string(suffix); + } + result[key] = std::move(info); + } + return result; +} + +Report workgroup_report(const Json& setting, const Json& resolved) { + return { + {"method", setting.is_string() ? setting.get() : "explicit"}, + {"value", resolved}}; +} + +std::string compact_tensor(const Report& tensor) { + const std::string storage = tensor.value("type", "buffer"); + std::string prefix = storage; + if (storage == "buffer") { + prefix = "b"; + } else if (storage == "texture2d") { + prefix = "t2"; + } else if (storage == "texture3d") { + prefix = "t3"; + } else if (storage == "constant") { + prefix = "c"; + } + const std::string dtype = tensor.at("dtype"); + std::string suffix = dtype; + if (dtype == "float32") { + suffix = "f32"; + } else if (dtype == "float16") { + suffix = "f16"; + } else if (dtype == "int32") { + suffix = "i32"; + } else if (dtype == "int8") { + suffix = "i8"; + } else if (dtype == "uint8") { + suffix = "u8"; + } else if (dtype == "int8x4") { + suffix = "i8x4"; + } + if (storage != "constant") { + suffix += "." + tensor.value("layout", "width"); + } + return prefix + tensor.at("sizes").dump() + suffix; +} + +std::string compact_input_value(const Report& value) { + if (value.is_number_float()) { + std::ostringstream output; + output << std::setprecision(3) << value.get(); + return output.str(); + } + if (value.is_array()) { + std::string output = "["; + for (size_t i = 0; i < value.size(); ++i) { + if (i > 0) { + output += ','; + } + output += compact_input_value(value.at(i)); + } + return output + ']'; + } + return value.dump(); +} + +std::string table_inputs(const Report& report, bool op) { + std::string inputs; + auto append = [&](const std::string& value) { + if (!inputs.empty()) { + inputs += ", "; + } + inputs += value; + }; + if (op) { + for (const auto& arg : report.at("args")) { + if (arg.contains("tensor")) { + if (arg.at("tensor").value("access", "read") != "write") { + append(compact_tensor(arg.at("tensor"))); + } + } else { + append( + compact_input_value(arg.is_object() ? arg.begin().value() : arg)); + } + } + } else { + auto append_blocks = [&](const Report& blocks) { + for (const auto& block : blocks) { + append(compact_input_value(block.at("data"))); + } + }; + for (const auto& binding : report.at("bindings")) { + if (binding.contains("sizes") && + binding.value("access", "read") != "write") { + append(compact_tensor(binding)); + } else if (binding.value("type", "") == "uniform") { + append_blocks(binding.at("values")); + } + } + if (report.contains("push_constants")) { + append_blocks(report.at("push_constants")); + } + } + return inputs.empty() ? "-" : inputs; +} + +std::string table_latency(const Report& stats, const char* key) { + if (stats.is_null()) { + return "N/A"; + } + std::ostringstream output; + output << std::fixed << std::setprecision(3) << stats.at(key).get(); + return output.str(); +} + +std::string table_gpu_throughput(const Report& report, const char* metric) { + if (!report.contains("throughput") || + !report.at("throughput").contains("gpu_timestamp")) { + return "N/A"; + } + const Report& rates = report.at("throughput").at("gpu_timestamp"); + return rates.contains(metric) ? table_latency(rates.at(metric), "mean") + : "N/A"; +} + +constexpr std::array kObservedGpuClockFields = { + "gpu_batch_observed_gpu_clock_freq_MHz", + "wall_time_batch_observed_gpu_clock_freq_MHz", + "shader_profile_batch_observed_gpu_clock_freq_MHz"}; + +void observe_gpu_clock_range( + std::array, 2>& range, + const Report& samples) { + if (!samples.is_array()) + return; + for (const auto& pair : samples) { + if (!pair.is_array()) + continue; + for (const auto& observation : pair) { + if (!observation.is_number()) + continue; + const double frequency = observation.get(); + if (!std::isfinite(frequency) || frequency < 0) + continue; + range[0] = range[0] ? std::min(*range[0], frequency) : frequency; + range[1] = range[1] ? std::max(*range[1], frequency) : frequency; + } + } +} + +std::array table_gpu_clock_range( + const Report& report, + bool profile_only = false) { + std::array, 2> range{}; + const size_t first = profile_only ? kObservedGpuClockFields.size() - 1 : 0; + for (size_t i = first; i < kObservedGpuClockFields.size(); ++i) { + const char* field = kObservedGpuClockFields[i]; + if (report.contains(field)) + observe_gpu_clock_range(range, report.at(field)); + } + std::array result{"N/A", "N/A"}; + for (size_t i = 0; i < result.size(); ++i) { + if (range[i]) { + std::ostringstream output; + output << std::setprecision(9) << *range[i]; + result[i] = output.str(); + } + } + return result; +} + +std::string table_text(std::string text, size_t max_chars = 0) { + for (char& ch : text) { + if (ch == '\n' || ch == '\r' || ch == '\t') { + ch = ' '; + } + } + if (max_chars != 0 && text.size() > max_chars) { + const size_t indentation = std::min(text.find_first_not_of(' '), max_chars); + const size_t dots = std::min(3, max_chars - indentation); + text = text.substr(0, max_chars - dots) + std::string(dots, '.'); + } + return text; +} + +std::string table_route(const Report& report) { + if (!report.contains("route")) { + return "-"; + } + const Report& route = report.at("route"); + const Report& requested = route.at("requested"); + const Report& resolved = route.at("resolved"); + const Report& selected = resolved.is_null() ? requested : resolved; + const std::string requested_name = requested.at("name"); + const std::string selected_name = selected.at("name"); + std::string label = requested_name == "auto" && selected_name != "auto" + ? "auto->" + selected_name + : selected_name; + const Report selected_options = selected.value("options", Report::object()); + if (selected_options.contains("intermediate_storage")) { + label += "[" + + selected_options.at("intermediate_storage").get() + "]"; + } + return table_text(label, 32); +} + +} // namespace + +std::string format_report(const Report& report) { + std::ostringstream output; + write_report_value(output, report, 0); + return output.str(); +} + +std::string format_condensed_report(const Report& report) { + auto condense_case = [](const Report& original) { + if (!original.is_object()) + return original; + Report condensed = Report::object(); + for (const auto& item : original.items()) { + const std::string& key = item.key(); + const Report& value = item.value(); + const bool batch_timing = key == "gpu_timestamp_operator_us" || + key == "gpu_timestamp_batch_us" || key == "wall_time_operator_us" || + key == "wall_time_batch_us" || key == "wall_time_shader_us"; + if (key == "args") { + condensed["args_str"] = table_text(table_inputs(original, true)); + } else if (batch_timing && value.is_array()) { + const Report stats = + latency_statistics(value.get>()); + condensed[key] = stats.is_null() ? Report() + : Report{ + {"min", stats.at("min")}, + {"median", stats.at("p50")}, + {"max", stats.at("max")}}; + } else if ( + std::find( + kObservedGpuClockFields.begin(), + kObservedGpuClockFields.end(), + key) != kObservedGpuClockFields.end()) { + std::array, 2> range{}; + observe_gpu_clock_range(range, value); + condensed[key] = range[0] + ? Report{{"min", *range[0]}, {"max", *range[1]}} + : Report(); + } else { + condensed[key] = value; + } + } + return condensed; + }; + if (!report.is_array()) + return format_report(condense_case(report)); + Report condensed = Report::array(); + for (const auto& item : report) + condensed.push_back(condense_case(item)); + return format_report(condensed); +} + +std::string format_table(const Report& reports, const TableOptions& options) { + const bool show_route = + std::any_of(reports.begin(), reports.end(), [](const Report& report) { + return report.contains("route"); + }); + const bool show_validation = + std::any_of(reports.begin(), reports.end(), [](const Report& report) { + return report.contains("validation"); + }); + const bool show_clocks = + std::any_of(reports.begin(), reports.end(), [](const Report& report) { + return std::any_of( + kObservedGpuClockFields.begin(), + kObservedGpuClockFields.end(), + [&](const char* field) { return report.contains(field); }); + }); + auto has_throughput = [&](const char* metric) { + return std::any_of( + reports.begin(), reports.end(), [&](const Report& report) { + return report.contains("throughput") && + report.at("throughput").contains(metric); + }); + }; + std::vector columns = {0}; + if (show_route) { + columns.push_back(9); + } + columns.insert(columns.end(), {1, 2, 3, 4, 5}); + if (show_clocks) { + columns.insert(columns.end(), {10, 11}); + } + if (show_validation) { + columns.push_back(6); + } + if (has_throughput("memory_bytes")) { + columns.push_back(7); + } + if (has_throughput("compute_flops")) { + columns.push_back(8); + } + using Row = std::array; + std::vector rows = { + {"Shader / operator", + "GWG", + "LWG", + "Inputs", + "Mean GPU us", + "P90 GPU us", + "Validation", + "GPU GiB/s", + "GPU GFLOPS", + "Route", + "Min GPU MHz", + "Max GPU MHz"}}; + std::vector case_rows; + std::vector errors; + for (const auto& report : reports) { + const bool op = report.contains("operator_name"); + const bool check = report.contains("check_name"); + const std::string name = report.at( + check ? "check_name" + : op ? "operator_name" + : "shader_name"); + const std::string suffix = report.contains("sweep_index") + ? " [" + std::to_string(report.at("sweep_index").get()) + "]" + : ""; + const bool skipped = report.value("status", "") == "SKIP"; + const std::string label = table_text( + name + suffix + (skipped ? " (SKIP)" : ""), options.name_max_chars); + const std::string route = table_route(report); + const std::string validation = !skipped && report.contains("validation") + ? table_text(report.at("validation").at("status").get()) + : "SKIP"; + case_rows.push_back(rows.size()); + if (skipped) { + const std::string inputs = report.contains(op ? "args" : "bindings") + ? table_inputs(report, op) + : "-"; + rows.push_back( + {label, + "-", + "-", + table_text(inputs, options.inputs_max_chars), + "N/A", + "N/A", + "SKIP", + "N/A", + "N/A", + route, + "N/A", + "N/A"}); + errors.push_back(table_text( + "SKIP " + report.value("case_name", name) + suffix + ": " + + report.value("skip_reason", ""))); + continue; + } + if (report.contains("error")) { + rows.push_back( + {label, + "-", + "-", + "ERROR", + "N/A", + "N/A", + validation, + "N/A", + "N/A", + route, + "N/A", + "N/A"}); + errors.push_back(table_text( + "ERROR " + report.value("case_name", name) + suffix + ": " + + report.at("error").get())); + continue; + } + if (check) { + rows.push_back( + {label, + "-", + "-", + "-", + "N/A", + "N/A", + validation, + "N/A", + "N/A", + "-", + "N/A", + "N/A"}); + if (report.at("validation").value("status", "") == "FAIL") + errors.push_back(table_text( + "FAIL " + report.value("case_name", name) + ": " + + report.at("validation").value("error", "Case check failed"))); + continue; + } + const auto& timing = + report.at(op ? "gpu_timestamp_operator_us" : "gpu_timestamp_shader_us"); + const Report stats = timing.is_array() + ? latency_statistics(timing.get>()) + : timing; + const auto clocks = table_gpu_clock_range(report); + rows.push_back( + {label, + op ? "-" : report.at("gwg_size").at("value").dump(), + op ? "-" : report.at("lwg_size").at("value").dump(), + table_text(table_inputs(report, op), options.inputs_max_chars), + table_latency(stats, "mean"), + table_latency(stats, "p90"), + validation, + table_gpu_throughput(report, "memory_gib_per_s"), + table_gpu_throughput(report, "compute_gflops"), + route, + clocks[0], + clocks[1]}); + if (op && report.contains("shader_breakdown")) { + const auto profile_clocks = table_gpu_clock_range(report, true); + for (const auto& shader : report.at("shader_breakdown")) { + const auto& shader_stats = shader.at("gpu_timestamp_shader_us"); + rows.push_back( + {table_text( + " " + shader.at("shader_name").get(), + options.name_max_chars), + shader.at("gwg_size").at("value").dump(), + shader.at("lwg_size").at("value").dump(), + "-", + table_latency(shader_stats, "mean"), + table_latency(shader_stats, "p90"), + "-", + table_gpu_throughput(shader, "memory_gib_per_s"), + table_gpu_throughput(shader, "compute_gflops"), + "-", + profile_clocks[0], + profile_clocks[1]}); + } + } + } + std::array widths{}; + for (const auto& row : rows) { + for (size_t i : columns) { + widths[i] = std::max(widths[i], row[i].size()); + } + } + const size_t line_width = + std::accumulate(widths.begin(), widths.end(), size_t{0}) + + 2 * (columns.size() - 1); + const std::string separator(line_width, '-'); + std::ostringstream output; + for (size_t r = 0; r < rows.size(); ++r) { + if (r > 0) { + output << '\n'; + } + if (std::binary_search(case_rows.begin(), case_rows.end(), r)) { + output << separator << '\n'; + } + for (size_t i : columns) { + if (i != columns.front()) { + output << " "; + } + output << (i < 4 || i == 6 || i == 9 ? std::left : std::right) + << std::setw(static_cast(widths[i])) << rows[r][i]; + } + } + for (const auto& error : errors) { + output << '\n' << error; + } + return output.str(); +} + +Report latency_statistics(std::vector samples) { + if (samples.empty()) { + return nullptr; + } + std::sort(samples.begin(), samples.end()); + return { + {"min", samples.front()}, + {"max", samples.back()}, + {"mean", + std::accumulate(samples.begin(), samples.end(), 0.0) / samples.size()}, + {"p50", percentile(samples, 0.5)}, + {"p90", percentile(samples, 0.9)}, + {"p95", percentile(samples, 0.95)}}; +} + +Report measurement_report( + uint32_t repetitions, + bool operator_case, + const std::vector& gpu_batches, + const std::vector& wall_batches) { + if (repetitions == 0) { + throw std::invalid_argument("Repetitions must be positive"); + } + std::vector gpu_average = gpu_batches; + std::vector wall_average = wall_batches; + for (double& value : gpu_average) { + value /= repetitions; + } + for (double& value : wall_average) { + value /= repetitions; + } + const Report gpu_latency = operator_case + ? Report(gpu_average) + : latency_statistics(std::move(gpu_average)); + return { + {operator_case ? "gpu_timestamp_operator_us" : "gpu_timestamp_shader_us", + gpu_latency}, + {"gpu_timestamp_batch_us", gpu_batches}, + {operator_case ? "wall_time_operator_us" : "wall_time_shader_us", + wall_average}, + {"wall_time_batch_us", wall_batches}}; +} + +Report throughput_report( + const Json& spec, + const WorkAmounts& work, + uint32_t repetitions, + const std::vector& gpu_batches, + const std::vector& wall_batches) { + const bool memory = + spec.is_object() && spec.value("show_memory_throughput", false); + const bool compute = + spec.is_object() && spec.value("show_compute_throughput", false); + Report result = Report::object(); + if (!memory && !compute) { + return result; + } + if (repetitions == 0) { + throw std::invalid_argument("Repetitions must be positive"); + } + auto set_work = [&](const char* key, const std::optional& amount) { + if (amount && (!std::isfinite(*amount) || *amount < 0)) { + throw std::invalid_argument("Work counts must be finite and nonnegative"); + } + result[key] = amount ? Report(*amount) : Report(); + }; + if (memory) { + set_work("memory_bytes", work.memory_bytes); + } + if (compute) { + set_work("compute_flops", work.compute_flops); + } + auto metrics = [&](const std::vector& batches, bool statistics) { + Report output = Report::object(); + auto add_metric = [&](const char* key, + const std::optional& amount, + double units) { + if (!amount) { + output[key] = nullptr; + return; + } + Report rates = Report::array(); + std::vector samples; + bool valid = true; + for (double batch_us : batches) { + const double invocation_us = batch_us / repetitions; + if (invocation_us <= 0 || !std::isfinite(invocation_us)) { + rates.push_back(nullptr); + valid = false; + continue; + } + const double rate = *amount / units * (1e6 / invocation_us); + if (std::isfinite(rate)) { + rates.push_back(rate); + samples.push_back(rate); + } else { + rates.push_back(nullptr); + valid = false; + } + } + output[key] = statistics + ? (valid ? latency_statistics(std::move(samples)) : Report()) + : std::move(rates); + }; + if (memory) { + add_metric("memory_gib_per_s", work.memory_bytes, 1073741824.0); + add_metric("memory_mib_per_s", work.memory_bytes, 1048576.0); + } + if (compute) { + add_metric("compute_gflops", work.compute_flops, 1e9); + } + return output; + }; + result["gpu_timestamp"] = metrics(gpu_batches, true); + result["gpu_timestamp_batch"] = result.at("gpu_timestamp"); + result["wall_time"] = metrics(wall_batches, false); + result["wall_time_batch"] = result.at("wall_time"); + if (!work.diagnostics.empty()) { + result["diagnostics"] = work.diagnostics; + } + return result; +} + +Report shader_report( + const Json& spec, + const Json& gwg, + const Json& lwg, + const Json& actual_dtypes, + const Report& constants) { + Report report = case_header(spec); + report["gwg_size"] = workgroup_report(spec.at("gwg_size"), gwg); + report["lwg_size"] = workgroup_report(spec.at("lwg_size"), lwg); + report["bindings"] = describe_bindings(spec, actual_dtypes); + report["specialization_constants"] = constants; + if (spec.contains("push_constants")) { + report["push_constants"] = spec.at("push_constants"); + } + return report; +} + +Report shader_breakdown_report( + const Report& dispatches, + const std::vector>& samples, + uint32_t repetitions, + const Json& throughput_flags, + const std::vector& work) { + if (repetitions == 0) { + throw std::invalid_argument("Repetitions must be positive"); + } + for (const auto& sample : samples) { + if (sample.size() != dispatches.size() * repetitions) { + throw std::invalid_argument( + "Shader timestamp count changed between runs"); + } + } + Report result = dispatches; + if (!work.empty() && work.size() != dispatches.size()) { + throw std::invalid_argument( + "Shader work count does not match dispatch count"); + } + for (size_t i = 0; i < dispatches.size(); ++i) { + std::vector durations; + durations.reserve(samples.size()); + for (const auto& sample : samples) { + double total = 0; + for (uint32_t r = 0; r < repetitions; ++r) { + total += sample[r * dispatches.size() + i]; + } + durations.push_back(total / repetitions); + } + result[i]["gpu_timestamp_shader_us"] = latency_statistics(durations); + Report throughput = throughput_report( + throughput_flags, + work.empty() ? WorkAmounts{} : work[i], + 1, + durations, + {}); + if (!throughput.empty()) { + throughput.erase("gpu_timestamp_batch"); + throughput.erase("wall_time"); + throughput.erase("wall_time_batch"); + result[i]["throughput"] = std::move(throughput); + } + } + return result; +} + +Report operator_report( + const Json& spec, + const Json& actual_dtypes, + const Json& actual_accesses) { + Report report = case_header(spec); + report["args"] = Report::array(); + for (const Json& arg : spec.at("args")) { + if (arg.contains("tensor")) { + const std::string name = arg.at("tensor"); + const Json& tensor = spec.at("tensors").at(name); + Report description = {{"name", name}}; + description.update(tensor_description( + tensor, actual_dtypes.value(name, tensor.value("dtype", "float32")))); + description["access"] = + arg.value("access", actual_accesses.value(name, Json("read"))); + report["args"].push_back({{"tensor", std::move(description)}}); + } else { + report["args"].push_back(Report(arg)); + } + } + return report; +} + +Report error_report(const Json& spec, const std::string& error) { + Report report = case_header(spec); + report["error"] = error; + Report throughput = throughput_report( + spec, + {std::nullopt, std::nullopt, {"No timing measurements"}}, + 1, + {}, + {}); + if (!throughput.empty()) { + report["throughput"] = std::move(throughput); + } + return report; +} + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/Report.h b/backends/vulkan/test/ops/benchmark_v2/framework/Report.h new file mode 100644 index 00000000000..e4751b1f6f9 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/Report.h @@ -0,0 +1,61 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include + +#include +#include +#include +#include + +namespace vkcompute::shader_profiling { + +using Report = nlohmann::ordered_json; + +struct TableOptions { + size_t name_max_chars = 32; + size_t inputs_max_chars = 64; +}; + +std::string format_report(const Report& report); +std::string format_condensed_report(const Report& report); +std::string format_table( + const Report& reports, + const TableOptions& options = {}); +Report latency_statistics(std::vector samples); +Report measurement_report( + uint32_t repetitions, + bool operator_case, + const std::vector& gpu_batches, + const std::vector& wall_batches); +Report throughput_report( + const Json& spec, + const WorkAmounts& work, + uint32_t repetitions, + const std::vector& gpu_batches, + const std::vector& wall_batches); +Report shader_breakdown_report( + const Report& dispatches, + const std::vector>& samples, + uint32_t repetitions = 1, + const Json& throughput_flags = Json::object(), + const std::vector& work = {}); +Report shader_report( + const Json& spec, + const Json& gwg, + const Json& lwg, + const Json& actual_dtypes, + const Report& constants); +Report operator_report( + const Json& spec, + const Json& actual_dtypes, + const Json& actual_accesses = Json::object()); +Report error_report(const Json& spec, const std::string& error); + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/RouteRegistry.cpp b/backends/vulkan/test/ops/benchmark_v2/framework/RouteRegistry.cpp new file mode 100644 index 00000000000..c1af2b3ac0b --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/RouteRegistry.cpp @@ -0,0 +1,95 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include + +namespace vkcompute::shader_profiling { + +RouteBuildFunction RouteRegistry::operator_route( + const std::string& op, + const std::string& route) const { + const auto it = operators_.find(op); + if (it == operators_.end() || !it->second.count(route)) { + throw std::invalid_argument("Unknown route for " + op + ": " + route); + } + return it->second.at(route); +} + +RouteInspectFunction RouteRegistry::operator_inspector( + const std::string& op) const { + const auto it = inspectors_.find(op); + return it == inspectors_.end() ? nullptr : it->second; +} + +void RouteRegistry::register_operator_route( + const std::string& op, + const std::string& route, + RouteBuildFunction function) { + if (op.empty() || route.empty() || route == "auto" || !function) { + throw std::invalid_argument( + "Route requires operator/route names and a function; auto is reserved"); + } + if (!operators_[op].emplace(route, function).second) { + throw std::invalid_argument("Duplicate route for " + op + ": " + route); + } +} + +void RouteRegistry::register_operator_inspector( + const std::string& op, + RouteInspectFunction function) { + if (op.empty() || !function) { + throw std::invalid_argument("Route inspector requires a name and function"); + } + if (!inspectors_.emplace(op, function).second) { + throw std::invalid_argument("Duplicate route inspector: " + op); + } +} + +RouteRegistry& route_registry() { + static RouteRegistry registry; + return registry; +} + +Json build_operator_route( + const Json& spec, + ComputeGraph& graph, + const std::vector& args) { + const std::string op = spec.at("operator"); + const Json requested = + spec.value("route", Json{{"name", "auto"}, {"options", Json::object()}}); + const std::string route = requested.at("name"); + const Json& options = requested.at("options"); + auto& registry = route_registry(); + Json resolved; + if (route == "auto") { + if (!options.empty()) { + throw std::invalid_argument("auto route does not accept options"); + } + const size_t first = graph.execute_nodes().size(); + VK_GET_OP_FN(op)(graph, args); + if (const auto inspector = registry.operator_inspector(op)) { + resolved = inspector(graph, args, first, graph.execute_nodes().size()); + } + } else { + const auto builder = registry.operator_route(op, route); + resolved = builder(graph, args, options); + } + if (route != "auto" && resolved.is_null()) { + throw std::invalid_argument("Route builder returned no resolved metadata"); + } + if (!resolved.is_null() && + (!resolved.is_object() || !resolved.contains("name") || + !resolved.at("name").is_string() || + resolved.at("name").get().empty() || + !resolved.contains("options") || !resolved.at("options").is_object())) { + throw std::invalid_argument("Route returned invalid resolved metadata"); + } + return {{"requested", requested}, {"resolved", resolved}}; +} + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/RouteRegistry.h b/backends/vulkan/test/ops/benchmark_v2/framework/RouteRegistry.h new file mode 100644 index 00000000000..a583bba9075 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/RouteRegistry.h @@ -0,0 +1,77 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include + +#include +#include +#include +#include +#include + +namespace vkcompute::shader_profiling { + +class UnsupportedRoute : public std::runtime_error { + public: + using std::runtime_error::runtime_error; +}; + +using RouteBuildFunction = + Json (*)(ComputeGraph&, const std::vector&, const Json&); +using RouteInspectFunction = + Json (*)(ComputeGraph&, const std::vector&, size_t, size_t); + +class RouteRegistry final { + private: + std::unordered_map< + std::string, + std::unordered_map> + operators_; + std::unordered_map inspectors_; + + public: + RouteBuildFunction operator_route( + const std::string& op, + const std::string& route) const; + RouteInspectFunction operator_inspector(const std::string& op) const; + void register_operator_route( + const std::string& op, + const std::string& route, + RouteBuildFunction function); + void register_operator_inspector( + const std::string& op, + RouteInspectFunction function); +}; + +class RouteRegisterInit final { + public: + explicit RouteRegisterInit(void (*initialize)()) { + initialize(); + } +}; + +RouteRegistry& route_registry(); +Json build_operator_route( + const Json& spec, + ComputeGraph& graph, + const std::vector& args); + +} // namespace vkcompute::shader_profiling + +#define REGISTER_OPERATOR_ROUTE(op, route, function) \ + ::vkcompute::shader_profiling::route_registry().register_operator_route( \ + #op, #route, &function) +#define REGISTER_ROUTE_INSPECTOR(op, function) \ + ::vkcompute::shader_profiling::route_registry().register_operator_inspector( \ + #op, &function) +#define REGISTER_OPERATOR_ROUTES \ + static void register_routes(); \ + static const ::vkcompute::shader_profiling::RouteRegisterInit route_reg( \ + ®ister_routes); \ + static void register_routes() diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/ShaderBenchmark.cpp b/backends/vulkan/test/ops/benchmark_v2/framework/ShaderBenchmark.cpp new file mode 100644 index 00000000000..84c577a3758 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/ShaderBenchmark.cpp @@ -0,0 +1,302 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace vkcompute::shader_profiling { + +namespace { + +std::vector packed_blocks(const Json& blocks) { + std::vector bytes; + for (const Json& block : blocks) { + if (block.value("dtype", "float32") == "float16") { + throw std::invalid_argument("Parameter blocks do not support float16"); + } + const auto data = make_data(block, block.at("data").size()); + bytes.insert(bytes.end(), data.begin(), data.end()); + } + if (bytes.size() % 4 != 0) { + throw std::invalid_argument( + "Parameter blocks must be padded to a multiple of four bytes"); + } + return bytes; +} + +} // namespace + +Report benchmark_shader( + const Json& spec, + const TensorData& host_data, + bool validation_only) { + GraphConfig config; + config.force_resize = true; + ComputeGraph graph(config); + const auto tensors = create_tensors(spec, graph, host_data, validation_only); + + const std::string shader_name = spec.at("shader"); + vkapi::ShaderInfo shader = VK_KERNEL_FROM_STR(shader_name); + // JSON describes shader invocations, rather than logical output elements. + shader.out_tile_size = LocalWorkGroup(1, 1, 1); + api::Context& context = *graph.context(); + context.check_device_capabilities(shader); + const Json& bindings = spec.at("bindings"); + if (bindings.size() != shader.kernel_layout.size()) { + throw std::invalid_argument( + "Binding count does not match shader signature"); + } + std::vector uniforms; + std::map params; + std::map image_bindings; + for (size_t i = 0; i < bindings.size(); ++i) { + const Json& binding = bindings.at(i); + const VkDescriptorType expected = shader.kernel_layout.at(i); + if (binding.contains("tensor")) { + const ValueRef tensor = + tensors.at(binding.at("tensor").get()); + const bool buffer = graph.is_buffer_storage(tensor); + const std::string access = binding.value("access", "read"); + if ((buffer && expected != VK_DESCRIPTOR_TYPE_STORAGE_BUFFER) || + (!buffer && expected != VK_DESCRIPTOR_TYPE_STORAGE_IMAGE && + expected != VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER) || + (expected == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER && + access != "read")) { + throw std::invalid_argument( + "Tensor storage/access does not match shader binding " + + std::to_string(i)); + } + if (!buffer) { + const auto previous = image_bindings.emplace(tensor, expected); + if (!previous.second && previous.first->second != expected) { + throw std::invalid_argument( + "Sampled and storage bindings must use separate texture tensors"); + } + } + } else { + if (expected != VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER) { + throw std::invalid_argument( + "Expected uniform buffer at binding " + std::to_string(i)); + } + if (binding.contains("meta_ubo")) { + params.emplace( + i, + meta_ubo( + graph, + tensors.at(binding.at("meta_ubo").get()), + binding.value("field", "meta"))); + } else { + const auto bytes = packed_blocks(binding.at("uniform")); + if (bytes.empty()) { + throw std::invalid_argument("Uniform buffer must not be empty"); + } + uniforms.push_back( + context.adapter_ptr()->vma().create_uniform_buffer(bytes.size())); + vkapi::MemoryMap map(uniforms.back(), vkapi::kWrite); + std::memcpy(map.data(), bytes.data(), bytes.size()); + params.emplace(i, vkapi::BufferBindInfo(uniforms.back())); + } + } + } + vkapi::SpecVarList constants; + for (const Json& constant : + spec.value("specialization_constants", Json::array())) { + if (constant.contains("layout")) { + constants.append(vkapi::SpecVar(graph.hashed_layout_of( + tensors.at(constant.at("layout").get())))); + } else { + const std::string dtype = constant.value("dtype", "int32"); + if (dtype == "bool") { + constants.append(vkapi::SpecVar(constant.at("value").get())); + } else { + const auto bytes = + make_data({{"dtype", dtype}, {"data", {constant.at("value")}}}, 1); + if (bytes.size() != 4) { + throw std::invalid_argument( + "Specialization constants must be 32-bit or bool"); + } + vkapi::SpecVar value; + if (dtype == "float32") { + value.type = vkapi::SpecVar::Type::FLOAT; + } else if (dtype == "int32") { + value.type = vkapi::SpecVar::Type::INT; + } else if (dtype == "uint32") { + value.type = vkapi::SpecVar::Type::UINT; + } else { + throw std::invalid_argument( + "Unsupported specialization constant dtype"); + } + std::memcpy(&value.value, bytes.data(), 4); + constants.append(value); + } + } + } + const auto push_constants = + packed_blocks(spec.value("push_constants", Json::array())); + if (push_constants.size() > 128) { + throw std::invalid_argument( + "Prototype supports up to 128 bytes of push constants"); + } + const auto workgroups = pick_workgroup_sizes(spec, graph, shader, tensors); + const GlobalWorkGrid& gwg = workgroups.gwg; + const LocalWorkGroup& lwg = workgroups.lwg; + + const size_t input_nodes = graph.execute_nodes().size(); + std::map outputs; + if (validation_only) { + Json accesses = Json::object(); + for (const auto& binding : bindings) { + if (binding.contains("tensor") && + binding.value("access", "read") != "read") { + accesses[binding.at("tensor").get()] = + binding.at("access"); + } + } + outputs = add_validation_outputs(spec, graph, tensors, accesses); + } + + graph.update_descriptor_counts(shader, true); + graph.prepare(); + graph.prepack(); + graph.propagate_resize(); + if (validation_only) { + copy_validation_inputs(spec, graph, tensors, host_data); + } + + const VkDescriptorSetLayout shader_layout = + context.shader_layout_cache().retrieve(shader.kernel_layout); + const VkPipelineLayout pipeline_layout = + context.pipeline_layout_cache().retrieve( + shader_layout, static_cast(push_constants.size())); + vkapi::SpecVarList pipeline_constants = {SV(lwg[0]), SV(lwg[1]), SV(lwg[2])}; + pipeline_constants.append(constants); + const VkPipeline pipeline = context.pipeline_cache().retrieve( + {pipeline_layout, + context.shader_cache().retrieve(shader), + pipeline_constants, + vkapi::resolve_required_subgroup_size(shader, context.adapter_ptr())}); + + vkapi::VulkanFence fence(context.device()); + auto bind_resources = [&](vkapi::DescriptorSet& descriptors, + vkapi::PipelineBarrier& barrier) { + for (size_t i = 0; i < bindings.size(); ++i) { + const Json& binding = bindings.at(i); + if (binding.contains("tensor")) { + const std::string access = binding.value("access", "read"); + const vkapi::MemoryAccessFlags flags = access == "read" ? vkapi::kRead + : access == "write" ? vkapi::kWrite + : vkapi::kReadWrite; + graph.bind_value_to_descriptor_set( + tensors.at(binding.at("tensor").get()), + barrier, + flags, + descriptors, + static_cast(i)); + } else { + descriptors.bind(static_cast(i), params.at(i)); + } + } + }; + if (!validation_only) { + // Initial image layout transitions must not be replayed. + context.set_cmd(); + auto initial_descriptors = context.descriptor_pool().get_descriptor_set( + shader_layout, shader.kernel_layout); + vkapi::PipelineBarrier transitions{}; + bind_resources(initial_descriptors, transitions); + context.register_barrier(transitions); + context.submit_cmd_to_gpu(fence.get_submit_handle(), true); + fence.wait(); + } + + auto encode = [&](uint32_t count) { + auto descriptors = context.descriptor_pool().get_descriptor_set( + shader_layout, shader.kernel_layout); + vkapi::PipelineBarrier initial{}; + bind_resources(descriptors, initial); + context.register_barrier(initial); + for (uint32_t i = 0; i < count; ++i) { + context.extract_cmd().bind_pipeline(pipeline, pipeline_layout, lwg); + // Serialize repeats, including across submissions of the cached buffer. + vkapi::PipelineBarrier barrier{}; + barrier.stage = { + VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_HOST_BIT, + VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT}; + bind_resources(descriptors, barrier); + context.register_shader_dispatch( + descriptors, + barrier, + shader, + gwg, + lwg, + push_constants.data(), + static_cast(push_constants.size())); + } + }; + if (validation_only) { + context.set_cmd(); + for (size_t i = 0; i < input_nodes; ++i) { + graph.execute_nodes()[i]->encode(&graph); + } + encode(1); + for (size_t i = input_nodes; i < graph.execute_nodes().size(); ++i) { + graph.execute_nodes()[i]->encode(&graph); + } + context.submit_cmd_to_gpu(fence.get_submit_handle(), true); + fence.wait(); + } + const Json definitions = + spec.value("specialization_constants", Json::array()); + Report specialization = Report::object(); + for (uint32_t i = 0; i < constants.size(); ++i) { + const vkapi::SpecVar& value = constants.at(i); + Report entry; + switch (value.type) { + case vkapi::SpecVar::Type::FLOAT: + entry = {{"dtype", "float32"}, {"value", value.value.as_float}}; + break; + case vkapi::SpecVar::Type::INT: + entry = {{"dtype", "int32"}, {"value", value.value.as_int32}}; + break; + case vkapi::SpecVar::Type::UINT: + entry = {{"dtype", "uint32"}, {"value", value.value.as_uint32}}; + break; + case vkapi::SpecVar::Type::BOOL: + entry = {{"dtype", "bool"}, {"value", value.value.as_bool}}; + break; + } + if (definitions.at(i).contains("layout")) { + entry["layout"] = definitions.at(i).at("layout"); + } + specialization[std::to_string(i + 3)] = std::move(entry); + } + Report result = shader_report( + spec, + {gwg[0], gwg[1], gwg[2]}, + {lwg[0], lwg[1], lwg[2]}, + actual_tensor_dtypes(graph, tensors), + specialization); + if (validation_only) { + result["validation"] = + validate_outputs(spec, graph, tensors, outputs, host_data); + } else { + result.update(measure(spec, graph, encode, Json(result))); + } + return result; +} + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/ShaderBenchmark.h b/backends/vulkan/test/ops/benchmark_v2/framework/ShaderBenchmark.h new file mode 100644 index 00000000000..681558bc8eb --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/ShaderBenchmark.h @@ -0,0 +1,19 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include + +namespace vkcompute::shader_profiling { + +Report benchmark_shader( + const Json& spec, + const TensorData& host_data, + bool validation_only = false); + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/Sweep.cpp b/backends/vulkan/test/ops/benchmark_v2/framework/Sweep.cpp new file mode 100644 index 00000000000..017da8b2ac5 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/Sweep.cpp @@ -0,0 +1,268 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include +#include +#include + +namespace vkcompute::shader_profiling { +namespace { + +constexpr size_t kMaxSweepCases = 65536; + +void require(bool condition, const std::string& message) { + if (!condition) { + throw std::invalid_argument(message); + } +} + +Json substitute(const Json& value, const Json& variables) { + if (value.is_string()) { + const std::string name = value.get(); + if (name.compare(0, 2, "v_") == 0) { + require(variables.contains(name), "Undefined sweep variable: " + name); + return variables.at(name); + } + } + Json result = value; + if (result.is_structured()) { + for (auto& item : result) { + item = substitute(item, variables); + } + } + return result; +} + +std::vector combinations(const Json& variables, size_t max_cases) { + require(variables.is_object(), "sweep_variables must be an object"); + std::vector result{Json::object()}; + for (const auto& variable : variables.items()) { + const std::string& name = variable.key(); + require( + name.size() > 2 && name.compare(0, 2, "v_") == 0, + "Sweep variable names must start with v_"); + require( + variable.value().is_array() && !variable.value().empty(), + "Sweep variable choices must be a nonempty array: " + name); + require( + variable.value().size() <= max_cases / result.size(), + "Sweep must not exceed 65536 cases"); + std::vector next; + for (const Json& combination : result) { + for (const Json& value : variable.value()) { + substitute(value, Json::object()); + Json entry = combination; + entry[name] = value; + next.push_back(std::move(entry)); + } + } + result = std::move(next); + } + return result; +} + +Json normalize_route(Json route) { + if (route.is_string()) { + route = {{"name", route}}; + } + require(route.is_object(), "route must be a name or an object"); + for (const auto& field : route.items()) { + require( + field.key() == "name" || field.key() == "options", + "route accepts only name and options"); + } + require( + route.contains("name") && route.at("name").is_string() && + !route.at("name").get().empty(), + "route.name must be a nonempty string"); + route["options"] = route.value("options", Json::object()); + require(route.at("options").is_object(), "route.options must be an object"); + return route; +} + +} // namespace + +std::vector expand_operator_sweep(const Json& definition) { + require(!definition.contains("args"), "Use args_sweep instead of args"); + require( + definition.contains("args_sweep") && + definition.at("args_sweep").is_array() && + !definition.at("args_sweep").empty(), + "args_sweep must be a nonempty list of argument sets"); + require( + !(definition.contains("route") && definition.contains("route_sweep")), + "Specify only one of route and route_sweep"); + const bool has_route = + definition.contains("route") || definition.contains("route_sweep"); + Json routes = Json::array({nullptr}); + if (definition.contains("route_sweep")) { + routes = definition.at("route_sweep"); + require( + routes.is_array() && !routes.empty(), + "route_sweep must be a nonempty list of routes"); + } else if (definition.contains("route")) { + routes = Json::array({definition.at("route")}); + } + const Json& sweep = definition.at("args_sweep"); + require( + routes.size() <= kMaxSweepCases && + sweep.size() <= kMaxSweepCases / routes.size(), + "Sweep must not exceed 65536 cases"); + const auto choices = combinations( + definition.value("sweep_variables", Json::object()), + kMaxSweepCases / (sweep.size() * routes.size())); + Json common = definition; + common.erase("args_sweep"); + common.erase("sweep_variables"); + common.erase("route"); + common.erase("route_sweep"); + std::vector result; + result.reserve(sweep.size() * choices.size() * routes.size()); + for (size_t i = 0; i < sweep.size(); ++i) { + require( + sweep.at(i).is_array(), + "Each args_sweep entry must be an argument array"); + common["args"] = sweep.at(i); + for (const Json& choice : choices) { + Json selected = substitute(common, choice); + if (!choice.empty()) { + selected["args_sweep_index"] = i; + selected["sweep_variables"] = choice; + } + for (const Json& route : routes) { + Json entry = selected; + if (has_route) { + entry["route"] = normalize_route(substitute(route, choice)); + } + entry["sweep_index"] = result.size(); + result.push_back(std::move(entry)); + } + } + } + return result; +} + +namespace { + +std::vector expand_lwg_sweep(const Json& definition, size_t max_cases) { + require( + !(definition.contains("lwg_size") && + definition.contains("lwg_size_sweep")), + "Specify only one of lwg_size and lwg_size_sweep"); + Json sizes; + if (definition.contains("lwg_size_sweep")) { + const Json& sweep = definition.at("lwg_size_sweep"); + require(sweep.is_object(), "lwg_size_sweep must be an object"); + for (const auto& axis : sweep.items()) { + require( + axis.key() == "x" || axis.key() == "y" || axis.key() == "z", + "lwg_size_sweep accepts only x, y, z"); + } + std::array starts{1, 1, 1}; + std::array steps{1, 1, 1}; + std::array counts{1, 1, 1}; + size_t count = 1; + for (size_t i = 0; i < 3; ++i) { + const char* axis = i == 0 ? "x" : i == 1 ? "y" : "z"; + if (!sweep.contains(axis)) { + continue; + } + const Json& range = sweep.at(axis); + require( + range.is_array() && range.size() == 3, + std::string("lwg_size_sweep.") + axis + + " must be [start, stop, step]"); + for (const Json& value : range) { + require( + value.is_number_integer() && value.get() > 0 && + value.get() <= UINT32_MAX, + "Local workgroup range values must be positive uint32 integers"); + } + starts[i] = range.at(0).get(); + const uint32_t stop = range.at(1).get(); + steps[i] = range.at(2).get(); + require(starts[i] <= stop, "Local workgroup range start exceeds stop"); + const uint64_t length = (uint64_t(stop) - starts[i]) / steps[i] + 1; + require(length <= max_cases / count, "Sweep must not exceed 65536 cases"); + counts[i] = static_cast(length); + count *= counts[i]; + } + sizes = Json::array(); + for (size_t x = 0; x < counts[0]; ++x) { + for (size_t y = 0; y < counts[1]; ++y) { + for (size_t z = 0; z < counts[2]; ++z) { + sizes.push_back( + {starts[0] + uint64_t(x) * steps[0], + starts[1] + uint64_t(y) * steps[1], + starts[2] + uint64_t(z) * steps[2]}); + } + } + } + } else if ( + definition.contains("lwg_size") && definition.at("lwg_size").is_array() && + !definition.at("lwg_size").empty() && + definition.at("lwg_size").front().is_array()) { + sizes = definition.at("lwg_size"); + } else { + return {definition}; + } + require(sizes.size() <= max_cases, "Sweep must not exceed 65536 cases"); + std::vector result; + result.reserve(sizes.size()); + for (const Json& size : sizes) { + require( + size.is_array() && size.size() == 3, + "Each lwg_size sweep entry must contain three positive integers"); + Json entry = definition; + entry.erase("lwg_size_sweep"); + entry["lwg_size"] = size; + entry["sweep_index"] = result.size(); + result.push_back(std::move(entry)); + } + return result; +} + +} // namespace + +std::vector expand_shader_sweep(const Json& definition) { + if (!definition.contains("shader_sweep")) { + return expand_lwg_sweep(definition, kMaxSweepCases); + } + require( + !definition.contains("shader"), + "Specify only one of shader and shader_sweep"); + const Json& shaders = definition.at("shader_sweep"); + require( + shaders.is_array() && !shaders.empty(), + "shader_sweep must be a nonempty list of shader names"); + require( + shaders.size() <= kMaxSweepCases, "Sweep must not exceed 65536 cases"); + for (const Json& shader : shaders) { + require( + shader.is_string() && !shader.get().empty(), + "shader_sweep entries must be nonempty shader names"); + } + Json common = definition; + common.erase("shader_sweep"); + const auto workgroups = + expand_lwg_sweep(common, kMaxSweepCases / shaders.size()); + std::vector result; + result.reserve(shaders.size() * workgroups.size()); + for (const Json& shader : shaders) { + for (const Json& workgroup : workgroups) { + Json entry = workgroup; + entry["shader"] = shader; + entry["sweep_index"] = result.size(); + result.push_back(std::move(entry)); + } + } + return result; +} + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/Sweep.h b/backends/vulkan/test/ops/benchmark_v2/framework/Sweep.h new file mode 100644 index 00000000000..d3fbea7c8c9 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/Sweep.h @@ -0,0 +1,16 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include + +namespace vkcompute::shader_profiling { + +std::vector expand_operator_sweep(const Json& definition); +std::vector expand_shader_sweep(const Json& definition); + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/TensorUtils.cpp b/backends/vulkan/test/ops/benchmark_v2/framework/TensorUtils.cpp new file mode 100644 index 00000000000..89c526e832e --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/TensorUtils.cpp @@ -0,0 +1,181 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include + +#include +#include +#include +#include + +#include +#include +#include + +namespace vkcompute::shader_profiling { + +namespace { + +const std::map kScalarTypes = { + {"float32", vkapi::kFloat}, + {"float16", vkapi::kHalf}, + {"int32", vkapi::kInt}, + {"int8", vkapi::kChar}, + {"uint8", vkapi::kByte}, + {"int8x4", vkapi::kInt8x4}}; + +const std::map kMemoryLayouts = { + {"width", utils::kWidthPacked}, + {"height", utils::kHeightPacked}, + {"channels", utils::kChannelsPacked}, + {"4w", utils::kPackedInt8_4W}, + {"4c", utils::kPackedInt8_4C}, + {"4w4c", utils::kPackedInt8_4W4C}, + {"4h4w", utils::kPackedInt8_4H4W}, + {"4c1w", utils::kPackedInt8_4C1W}}; + +} // namespace + +vkapi::ScalarType scalar_type(const std::string& name) { + return kScalarTypes.at(name); +} + +vkapi::BufferBindInfo +meta_ubo(ComputeGraph& graph, ValueRef tensor, const std::string& field) { + if (field == "meta") { + return graph.meta_ubo(tensor); + } + if (field == "sizes") { + return graph.sizes_ubo(tensor); + } + if (field == "strides") { + return graph.strides_ubo(tensor); + } + if (field == "numel") { + return graph.numel_ubo(tensor); + } + if (field == "logical_limits") { + return graph.logical_limits_ubo(tensor); + } + throw std::invalid_argument("Unknown tensor metadata field: " + field); +} + +TensorData prepare_tensor_data(const Json& spec) { + const Json descriptions = spec.value("tensors", Json::object()); + auto host_data = make_tensor_data(descriptions); + for (const auto& item : descriptions.items()) { + const Json& tensor = item.value(); + if (tensor.value("storage", "buffer") != "constant" || + tensor.value("dtype", "float32") != "float16") { + continue; + } + auto& data = host_data.at(item.key()); + std::vector half_data(data.size() / 2); + for (size_t i = 0; i < data.size() / sizeof(float); ++i) { + float value; + std::memcpy(&value, data.data() + i * sizeof(float), sizeof(float)); + const executorch::runtime::etensor::Half half(value); + std::memcpy(half_data.data() + i * sizeof(half), &half, sizeof(half)); + } + data = std::move(half_data); + } + return host_data; +} + +std::map create_tensors( + const Json& spec, + ComputeGraph& graph, + const TensorData& host_data, + bool with_staging) { + std::map tensors; + const Json descriptions = spec.value("tensors", Json::object()); + for (const auto& item : descriptions.items()) { + const Json& tensor = item.value(); + const auto sizes = tensor.at("sizes").get>(); + const auto dtype = scalar_type(tensor.value("dtype", "float32")); + const auto& data = host_data.at(item.key()); + const std::string storage = tensor.value("storage", "buffer"); + const bool constant = storage == "constant"; + const vkapi::ScalarType source_dtype = dtype == vkapi::kHalf && !constant + ? vkapi::kFloat + : dtype == vkapi::kInt8x4 ? vkapi::kChar + : dtype; + const ValueRef source = + graph.add_tensorref(sizes, source_dtype, data.data()); + if (constant) { + tensors.emplace(item.key(), source); + continue; + } + const auto storage_type = storage == "buffer" ? utils::kBuffer + : storage == "texture2d" ? utils::kTexture2D + : utils::kTexture3D; + const ValueRef ref = graph.add_tensor( + sizes, + dtype, + storage_type, + kMemoryLayouts.at(tensor.value("layout", "width"))); + tensors.emplace(item.key(), ref); + if (with_staging) { + graph.set_input_tensor(ref, source_dtype); + } else if (dtype == vkapi::kInt8x4) { + add_prepack_int8x4_buffer_node(graph, source, ref); + } else { + add_prepack_standard_node(graph, source, ref); + } + } + return tensors; +} + +Json actual_tensor_dtypes( + ComputeGraph& graph, + const std::map& tensors) { + Json result = Json::object(); + for (const auto& tensor : tensors) { + const auto dtype = graph.dtype_of(tensor.second); + std::string name = vkapi::to_string(dtype); + for (const auto& entry : kScalarTypes) { + if (entry.second == dtype) { + name = entry.first; + break; + } + } + result[tensor.first] = name; + } + return result; +} + +Json dispatch_tensor_bindings(ComputeGraph& graph, const ExecuteNode& node) { + Json bindings = Json::object(); + size_t index = 0; + for (const auto& group : node.args()) { + for (ValueRef ref : group.refs) { + if (ref < 0 || !graph.val_is_tensor(ref)) { + ++index; + continue; + } + const std::string name = "value_" + std::to_string(ref); + const std::string storage = graph.is_buffer_storage(ref) ? "buffer" + : graph.storage_type_of(ref) == utils::kTexture2D ? "texture2d" + : "texture3d"; + const auto access = group.access; + bindings["binding_" + std::to_string(index)] = { + {"name", name}, + {"binding_index", index}, + {"type", storage}, + {"sizes", graph.sizes_of(ref)}, + {"dtype", actual_tensor_dtypes(graph, {{name, ref}}).at(name)}, + {"access", + access & vkapi::kWrite + ? (access & vkapi::kRead ? "read_write" : "write") + : "read"}}; + ++index; + } + } + return bindings; +} + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/TensorUtils.h b/backends/vulkan/test/ops/benchmark_v2/framework/TensorUtils.h new file mode 100644 index 00000000000..1c2c79211ea --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/TensorUtils.h @@ -0,0 +1,40 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include + +#include +#include +#include +#include +#include + +namespace vkcompute::shader_profiling { + +using TensorData = std::map>; + +TensorData prepare_tensor_data(const Json& spec); + +vkapi::ScalarType scalar_type(const std::string& name); + +vkapi::BufferBindInfo +meta_ubo(ComputeGraph& graph, ValueRef tensor, const std::string& field); + +std::map create_tensors( + const Json& spec, + ComputeGraph& graph, + const TensorData& host_data, + bool with_staging = false); + +Json actual_tensor_dtypes( + ComputeGraph& graph, + const std::map& tensors); +Json dispatch_tensor_bindings(ComputeGraph& graph, const ExecuteNode& node); + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/ThroughputRegistry.cpp b/backends/vulkan/test/ops/benchmark_v2/framework/ThroughputRegistry.cpp new file mode 100644 index 00000000000..4d456dc525e --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/ThroughputRegistry.cpp @@ -0,0 +1,280 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include +#include + +namespace vkcompute::shader_profiling { +namespace { + +void require(bool condition, const std::string& message) { + if (!condition) { + throw std::invalid_argument(message); + } +} + +void associate( + std::unordered_map& table, + const std::string& name, + WorkCountFunction function, + const std::string& kind) { + require( + !name.empty() && function, + "Work count registration requires a nonempty name and callable"); + require( + table.emplace(name, function).second, + "Duplicate " + kind + " work count association: " + name); +} + +WorkCountFunction lookup( + const std::unordered_map& table, + const std::string& name, + const std::string& kind) { + const auto entry = table.find(name); + require( + entry != table.end(), + "No " + kind + " work count registered for " + name); + return entry->second; +} + +WorkCountFunction lookup_shader( + const std::unordered_map& exact, + const std::unordered_map& prefixes, + const std::string& name, + const std::string& kind) { + const auto entry = exact.find(name); + if (entry != exact.end()) { + return entry->second; + } + WorkCountFunction function = nullptr; + size_t length = 0; + for (const auto& prefix : prefixes) { + if (prefix.first.size() > length && + name.compare(0, prefix.first.size(), prefix.first) == 0) { + function = prefix.second; + length = prefix.first.size(); + } + } + require( + function != nullptr, "No " + kind + " work count registered for " + name); + return function; +} + +bool tensor_type(const Json& tensor) { + const std::string type = tensor.value("type", ""); + return type == "buffer" || type == "texture2d" || type == "texture3d" || + type == "constant"; +} + +} // namespace + +WorkCountFunction ThroughputRegistry::operator_memory( + const std::string& name) const { + const auto function = lookup(operator_memory_, name, "operator memory"); + return function; +} +WorkCountFunction ThroughputRegistry::operator_compute( + const std::string& name) const { + const auto function = lookup(operator_compute_, name, "operator compute"); + return function; +} +WorkCountFunction ThroughputRegistry::shader_memory( + const std::string& name) const { + const auto function = lookup_shader( + shader_memory_, shader_memory_prefixes_, name, "shader memory"); + return function; +} +WorkCountFunction ThroughputRegistry::shader_compute( + const std::string& name) const { + const auto function = lookup_shader( + shader_compute_, shader_compute_prefixes_, name, "shader compute"); + return function; +} + +void ThroughputRegistry::register_operator_memory( + const std::string& name, + WorkCountFunction function) { + associate(operator_memory_, name, function, "operator memory"); +} +void ThroughputRegistry::register_operator_compute( + const std::string& name, + WorkCountFunction function) { + associate(operator_compute_, name, function, "operator compute"); +} +void ThroughputRegistry::register_shader_memory( + const std::string& name, + WorkCountFunction function) { + associate(shader_memory_, name, function, "shader memory"); +} +void ThroughputRegistry::register_shader_compute( + const std::string& name, + WorkCountFunction function) { + associate(shader_compute_, name, function, "shader compute"); +} + +void ThroughputRegistry::register_shader_memory_prefix( + const std::string& prefix, + WorkCountFunction function) { + associate(shader_memory_prefixes_, prefix, function, "shader memory prefix"); +} + +void ThroughputRegistry::register_shader_compute_prefix( + const std::string& prefix, + WorkCountFunction function) { + associate( + shader_compute_prefixes_, prefix, function, "shader compute prefix"); +} + +ThroughputRegistry& throughput_registry() { + static ThroughputRegistry registry; + return registry; +} + +WorkAmounts estimate_work(const Json& spec, bool memory, bool compute) { + WorkAmounts amounts; + const auto calculate = [&](bool memory_count) -> std::optional { + try { + require( + spec.contains("operator_name") != spec.contains("shader_name"), + "Work counts require exactly one operator_name or shader_name"); + const bool op = spec.contains("operator_name"); + const std::string name = spec.at(op ? "operator_name" : "shader_name"); + const auto& registry = throughput_registry(); + const auto function = op + ? (memory_count ? registry.operator_memory(name) + : registry.operator_compute(name)) + : (memory_count ? registry.shader_memory(name) + : registry.shader_compute(name)); + const double count = function(spec); + require( + std::isfinite(count) && count >= 0, + "Work count must be finite and nonnegative for " + name); + return count; + } catch (const std::exception& error) { + amounts.diagnostics.push_back( + std::string( + memory_count ? "Memory throughput: " : "Compute throughput: ") + + error.what()); + return std::nullopt; + } + }; + if (memory) { + amounts.memory_bytes = calculate(true); + } + if (compute) { + amounts.compute_flops = calculate(false); + } + return amounts; +} + +namespace throughput { + +double tensor_elements(const Json& tensor) { + return double(tensor_numel(tensor.at("sizes"))); +} + +double tensor_bytes(const Json& tensor) { + static const std::map bytes_per_element = { + {"float32", 4}, + {"float16", 2}, + {"int32", 4}, + {"int8", 1}, + {"uint8", 1}, + {"int8x4", 1}}; + const std::string dtype = tensor.at("dtype"); + const auto element_size = bytes_per_element.find(dtype); + require( + element_size != bytes_per_element.end(), + "Unknown tensor dtype for memory work count: " + dtype); + return tensor_elements(tensor) * element_size->second; +} + +double tensor_traffic(const Json& spec) { + struct Traffic { + double bytes; + unsigned access; + }; + std::map tensors; + const auto add = [&](const Json& tensor, const std::string& fallback_name) { + require( + tensor_type(tensor), "Unknown tensor storage for memory work count"); + const std::string access = tensor.at("access"); + require( + access == "read" || access == "write" || access == "read_write", + "Unknown tensor access for memory work count: " + access); + const unsigned flags = access == "read" ? 1 : access == "write" ? 2 : 3; + const std::string name = + tensor.value("ref", tensor.value("name", fallback_name)); + const double bytes = tensor_bytes(tensor); + auto entry = tensors.emplace(name, Traffic{bytes, flags}); + require( + entry.first->second.bytes == bytes, + "Aliased tensor byte counts disagree: " + name); + entry.first->second.access |= flags; + }; + if (spec.contains("operator_name")) { + const Json& args = spec.at("args"); + for (size_t i = 0; i < args.size(); ++i) { + if (args[i].contains("tensor")) { + add(args[i].at("tensor"), "arg_" + std::to_string(i)); + } + } + } else { + for (const auto& binding : spec.at("bindings").items()) { + if (tensor_type(binding.value()) || binding.value().contains("sizes")) { + add(binding.value(), binding.key()); + } + } + } + require(!tensors.empty(), "Memory work count requires tensor bindings"); + double total = 0; + for (const auto& tensor : tensors) { + const auto& traffic = tensor.second; + total += + traffic.bytes * ((traffic.access & 1) + ((traffic.access >> 1) & 1)); + } + return total; +} + +const Json& tensor_argument(const Json& spec, size_t index) { + const Json& tensor = spec.at("args").at(index).at("tensor"); + require( + tensor.is_object() && tensor.contains("sizes"), + "Work count requires resolved tensor argument metadata"); + return tensor; +} + +const Json& output_tensor(const Json& spec) { + if (spec.contains("operator_name")) { + return tensor_argument(spec, spec.at("args").size() - 1); + } + const Json* result = nullptr; + for (const auto& binding : spec.at("bindings").items()) { + const Json& tensor = binding.value(); + if (!tensor_type(tensor)) { + continue; + } + const std::string access = tensor.at("access"); + if (access == "write" || access == "read_write") { + if (!result || + tensor.value("binding_index", SIZE_MAX) < + result->value("binding_index", SIZE_MAX)) { + result = &tensor; + } + } + } + require( + result != nullptr, + "Compute work count requires an output tensor binding"); + return *result; +} + +} // namespace throughput + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/ThroughputRegistry.h b/backends/vulkan/test/ops/benchmark_v2/framework/ThroughputRegistry.h new file mode 100644 index 00000000000..43fdf3bf50d --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/ThroughputRegistry.h @@ -0,0 +1,105 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include + +#include +#include +#include +#include + +namespace vkcompute::shader_profiling { + +using WorkCountFunction = double (*)(const Json&); + +class ThroughputRegistry final { + private: + std::unordered_map operator_memory_; + std::unordered_map operator_compute_; + std::unordered_map shader_memory_; + std::unordered_map shader_compute_; + std::unordered_map shader_memory_prefixes_; + std::unordered_map shader_compute_prefixes_; + + public: + WorkCountFunction operator_memory(const std::string& name) const; + WorkCountFunction operator_compute(const std::string& name) const; + WorkCountFunction shader_memory(const std::string& name) const; + WorkCountFunction shader_compute(const std::string& name) const; + void register_operator_memory( + const std::string& name, + WorkCountFunction function); + void register_operator_compute( + const std::string& name, + WorkCountFunction function); + void register_shader_memory( + const std::string& name, + WorkCountFunction function); + void register_shader_compute( + const std::string& name, + WorkCountFunction function); + void register_shader_memory_prefix( + const std::string& prefix, + WorkCountFunction function); + void register_shader_compute_prefix( + const std::string& prefix, + WorkCountFunction function); +}; + +class ThroughputRegisterInit final { + public: + explicit ThroughputRegisterInit(void (*initialize)()) { + initialize(); + } +}; + +struct WorkAmounts { + std::optional memory_bytes; + std::optional compute_flops; + std::vector diagnostics; +}; + +ThroughputRegistry& throughput_registry(); +WorkAmounts estimate_work(const Json& resolved_case, bool memory, bool compute); + +namespace throughput { + +double tensor_elements(const Json& tensor); +double tensor_bytes(const Json& tensor); +double tensor_traffic(const Json& spec); +const Json& tensor_argument(const Json& spec, size_t index); +const Json& output_tensor(const Json& spec); + +} // namespace throughput + +} // namespace vkcompute::shader_profiling + +#define REGISTER_OPERATOR_MEMORY_TRAFFIC(name, function) \ + ::vkcompute::shader_profiling::throughput_registry() \ + .register_operator_memory(#name, &function) +#define REGISTER_OPERATOR_COMPUTE_FLOPS(name, function) \ + ::vkcompute::shader_profiling::throughput_registry() \ + .register_operator_compute(#name, &function) +#define REGISTER_SHADER_MEMORY(name, function) \ + ::vkcompute::shader_profiling::throughput_registry().register_shader_memory( \ + #name, &function) +#define REGISTER_SHADER_COMPUTE(name, function) \ + ::vkcompute::shader_profiling::throughput_registry() \ + .register_shader_compute(#name, &function) +#define REGISTER_SHADER_MEMORY_TRAFFIC(prefix, function) \ + ::vkcompute::shader_profiling::throughput_registry() \ + .register_shader_memory_prefix(#prefix, &function) +#define REGISTER_SHADER_COMPUTE_FLOPS(prefix, function) \ + ::vkcompute::shader_profiling::throughput_registry() \ + .register_shader_compute_prefix(#prefix, &function) + +#define REGISTER_THROUGHPUT_FUNCTIONS \ + static void register_throughput_functions(); \ + static const ::vkcompute::shader_profiling::ThroughputRegisterInit \ + throughput_reg(®ister_throughput_functions); \ + static void register_throughput_functions() diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/Timestamps.cpp b/backends/vulkan/test/ops/benchmark_v2/framework/Timestamps.cpp new file mode 100644 index 00000000000..d85e0800ac5 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/Timestamps.cpp @@ -0,0 +1,82 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include + +namespace vkcompute::shader_profiling { + +Timestamps::Timestamps(api::Context& context) + : device_(context.device()), + period_ns_(context.adapter_ptr()->timestamp_period()) { + uint32_t count = 0; + const VkPhysicalDevice physical = context.adapter_ptr()->physical_handle(); + vkGetPhysicalDeviceQueueFamilyProperties(physical, &count, nullptr); + std::vector families(count); + vkGetPhysicalDeviceQueueFamilyProperties(physical, &count, families.data()); + valid_bits_ = families.at(context.queue().family_index).timestampValidBits; + if (valid_bits_ > 0 && period_ns_ > 0) { + const VkQueryPoolCreateInfo info{ + VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO, + nullptr, + 0, + VK_QUERY_TYPE_TIMESTAMP, + 2, + 0}; + VK_CHECK(vkCreateQueryPool(device_, &info, nullptr, &pool_)); + } +} + +Timestamps::~Timestamps() { + if (pool_ != VK_NULL_HANDLE) { + vkDestroyQueryPool(device_, pool_, nullptr); + } +} + +bool Timestamps::available() const { + return pool_ != VK_NULL_HANDLE; +} + +double Timestamps::period_ns() const { + return period_ns_; +} + +uint32_t Timestamps::valid_bits() const { + return valid_bits_; +} + +void Timestamps::begin(const vkapi::CommandBuffer& cmd) const { + cmd.reset_querypool(pool_, 0, 2); + cmd.write_timestamp(pool_, 0); +} + +void Timestamps::end(const vkapi::CommandBuffer& cmd) const { + cmd.write_timestamp(pool_, 1); +} + +double Timestamps::elapsed_us() const { + uint64_t ticks[2]; + VK_CHECK(vkGetQueryPoolResults( + device_, + pool_, + 0, + 2, + sizeof(ticks), + ticks, + sizeof(uint64_t), + VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WAIT_BIT)); + return elapsed_us(ticks[0], ticks[1]); +} + +double Timestamps::elapsed_us(uint64_t start_ticks, uint64_t end_ticks) const { + const uint64_t mask = + valid_bits_ == 64 ? UINT64_MAX : (uint64_t(1) << valid_bits_) - 1; + return static_cast((end_ticks - start_ticks) & mask) * period_ns_ / + 1000.0; +} + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/Timestamps.h b/backends/vulkan/test/ops/benchmark_v2/framework/Timestamps.h new file mode 100644 index 00000000000..774d1a3f40b --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/Timestamps.h @@ -0,0 +1,37 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include + +#include + +namespace vkcompute::shader_profiling { + +class Timestamps final { + VkDevice device_; + VkQueryPool pool_ = VK_NULL_HANDLE; + double period_ns_; + uint32_t valid_bits_; + + public: + explicit Timestamps(api::Context& context); + Timestamps(const Timestamps&) = delete; + ~Timestamps(); + + Timestamps& operator=(const Timestamps&) = delete; + + bool available() const; + double period_ns() const; + uint32_t valid_bits() const; + void begin(const vkapi::CommandBuffer& cmd) const; + void end(const vkapi::CommandBuffer& cmd) const; + double elapsed_us() const; + double elapsed_us(uint64_t start_ticks, uint64_t end_ticks) const; +}; + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/Timing.cpp b/backends/vulkan/test/ops/benchmark_v2/framework/Timing.cpp new file mode 100644 index 00000000000..9e976a0b678 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/Timing.cpp @@ -0,0 +1,275 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace vkcompute::shader_profiling { + +namespace { + +using Clock = std::chrono::steady_clock; + +Report read_gpu_clock(GpuClock* clock) { + const auto frequency = clock ? clock->read_mhz() : std::nullopt; + return frequency ? Report(*frequency) : Report(); +} + +Report profile_operator_shaders( + const Json& spec, + ComputeGraph& graph, + const Timestamps& timestamps, + const std::function& encode_batch, + uint32_t count, + uint32_t query_limit, + uint32_t warmup, + uint32_t runs, + GpuClock* gpu_clock) { + auto& context = *graph.context(); + context.flush(); + context.initialize_querypool(); + context.set_shader_profiling_enabled(true); + context.set_cmd(true); + context.cmd_reset_querypool(); + const bool memory = spec.value("show_memory_throughput", false); + const bool compute = spec.value("show_compute_throughput", false); + std::vector work; + if (memory || compute) { + for (const auto& node : graph.execute_nodes()) { + const size_t before = + context.querypool().get_shader_timestamp_data().size(); + node->encode(&graph); + const auto records = context.querypool().get_shader_timestamp_data(); + const size_t emitted = + records.size() > before ? records.size() - before : 0; + for (size_t i = before; i < records.size(); ++i) { + if (emitted == 1) { + work.push_back(estimate_work( + {{"shader_name", records[i].kernel_name}, + {"gwg_size", + {{"method", "operator"}, {"value", records[i].metadata.gwg}}}, + {"lwg_size", + {{"method", "operator"}, {"value", records[i].metadata.lwg}}}, + {"bindings", dispatch_tensor_bindings(graph, *node)}}, + memory, + compute)); + } else { + work.push_back( + {std::nullopt, + std::nullopt, + {"Binding metadata unavailable for a node with multiple dispatches"}}); + } + } + } + } else { + encode_batch(1); + } + const size_t shaders_per_invocation = + context.querypool().get_shader_timestamp_data().size(); + if ((memory || compute) && work.size() != shaders_per_invocation) { + work.assign( + shaders_per_invocation, + {std::nullopt, std::nullopt, {"Shader binding metadata unavailable"}}); + } + if (shaders_per_invocation > query_limit / 2) { + throw UnsupportedRoute( + "Operator dispatch chain exceeds timestamp query capacity"); + } + count = shaders_per_invocation == 0 + ? 1u + : std::min( + count, + static_cast(query_limit / (2 * shaders_per_invocation))); + if (count > 1) { + context.flush(); + context.set_cmd(true); + context.cmd_reset_querypool(); + encode_batch(count); + } + vkapi::VulkanFence fence(context.device()); + auto replay = [&]() { + context.submit_cmd_to_gpu(fence.get_submit_handle()); + fence.wait(); + }; + for (uint32_t i = 0; i < warmup; ++i) { + replay(); + } + Report dispatches = Report::array(); + Report clock_samples = Report::array(); + std::vector> samples; + for (uint32_t i = 0; i < runs; ++i) { + const Report clock_start = read_gpu_clock(gpu_clock); + replay(); + if (gpu_clock) { + clock_samples.push_back({clock_start, read_gpu_clock(gpu_clock)}); + } + const auto shaders = context.querypool().get_shader_timestamp_ticks(); + std::vector durations; + for (size_t j = 0; j < shaders.size(); ++j) { + const auto& shader = shaders[j]; + if (i == 0 && j < shaders_per_invocation) { + dispatches.push_back( + {{"dispatch_index", j}, + {"shader_name", shader.kernel_name}, + {"gwg_size", + {{"method", "operator"}, {"value", shader.metadata.gwg}}}, + {"lwg_size", + {{"method", "operator"}, {"value", shader.metadata.lwg}}}}); + } + durations.push_back( + timestamps.elapsed_us(shader.start_ticks, shader.end_ticks)); + } + samples.push_back(std::move(durations)); + } + Report result = { + {"shader_profile_invocations_per_batch", count}, + {"shader_breakdown", + shader_breakdown_report(dispatches, samples, count, spec, work)}}; + if (gpu_clock) { + result["shader_profile_batch_observed_gpu_clock_freq_MHz"] = + std::move(clock_samples); + } + return result; +} + +} // namespace + +Report measure( + const Json& spec, + ComputeGraph& graph, + const std::function& encode_batch, + const Json& work_spec) { + api::Context& context = *graph.context(); + GpuClock* gpu_clock = nullptr; +#if defined(__ANDROID__) + GpuClock android_gpu_clock; + gpu_clock = &android_gpu_clock; +#endif + Report wall_clock_samples = Report::array(); + Report gpu_clock_samples = Report::array(); + Timestamps timestamps(context); + vkapi::VulkanFence fence(context.device()); + const WorkAmounts work = estimate_work( + work_spec, + spec.value("show_memory_throughput", false), + spec.value("show_compute_throughput", false)); + auto encode = [&](uint32_t count, bool timed) { + context.set_shader_profiling_enabled(false); + context.flush(); + context.set_cmd(true); + if (timed) { + timestamps.begin(context.extract_cmd()); + } + encode_batch(count); + if (timed) { + timestamps.end(context.extract_cmd()); + } + }; + auto replay = [&]() { + const auto start = Clock::now(); + context.submit_cmd_to_gpu(fence.get_submit_handle()); + fence.wait(); + return std::chrono::duration(Clock::now() - start) + .count(); + }; + auto measured_replay = [&](Report& clock_samples) { + const Report clock_start = read_gpu_clock(gpu_clock); + const double wall = replay(); + if (gpu_clock) { + clock_samples.push_back({clock_start, read_gpu_clock(gpu_clock)}); + } + return wall; + }; + + uint32_t count = 1; + const uint32_t limit = + spec.value("max_dispatches", spec.contains("operator") ? 200u : 65536u); + const double target_us = spec.value("target_ms", 50.0) * 1000; + for (;;) { + encode(count, false); + replay(); + const double wall = replay(); + if (wall >= target_us * 0.8 || count == limit) + break; + const uint32_t next = choose_dispatch_count(wall / count, target_us, limit); + count = std::min(limit, std::max(count + 1, std::min(next, count * 8))); + } + const uint32_t warmup = spec.value("warmup_runs", 3u); + const uint32_t runs = spec.value("benchmark_runs", 10u); + encode(count, false); + std::vector wall_samples; + std::vector gpu_samples; + for (uint32_t i = 0; i < warmup; ++i) { + replay(); + } + for (uint32_t i = 0; i < runs; ++i) { + wall_samples.push_back(measured_replay(wall_clock_samples)); + } + if (timestamps.available()) { + encode(count, true); + for (uint32_t i = 0; i < warmup; ++i) { + replay(); + } + for (uint32_t i = 0; i < runs; ++i) { + measured_replay(gpu_clock_samples); + gpu_samples.push_back(timestamps.elapsed_us()); + } + } + Report result = { + {"device", graph.device_name()}, + {"target_ms", target_us / 1000}, + {"max_dispatches", limit}, + {spec.contains("operator") ? "operator_invocations_per_batch" + : "dispatches_per_batch", + count}, + {"warmup_runs", warmup}, + {"benchmark_runs", runs}, + {"gpu_timestamps_available", timestamps.available()}, + {"timestamp_period_ns", timestamps.period_ns()}, + {"timestamp_valid_bits", timestamps.valid_bits()}}; + const Report measurements = measurement_report( + count, spec.contains("operator"), gpu_samples, wall_samples); + for (const auto& measurement : measurements.items()) { + result[measurement.key()] = measurement.value(); + if (gpu_clock && measurement.key() == "gpu_timestamp_batch_us") { + result["gpu_batch_observed_gpu_clock_freq_MHz"] = + std::move(gpu_clock_samples); + } + if (gpu_clock && measurement.key() == "wall_time_batch_us") { + result["wall_time_batch_observed_gpu_clock_freq_MHz"] = + std::move(wall_clock_samples); + } + } + Report throughput = + throughput_report(spec, work, count, gpu_samples, wall_samples); + if (!throughput.empty()) { + result["throughput"] = std::move(throughput); + } + if (spec.contains("operator") && timestamps.available()) { + result.update(profile_operator_shaders( + spec, + graph, + timestamps, + encode_batch, + count, + graph.graphconfig().context_config.query_pool_config.max_query_count, + warmup, + runs, + gpu_clock)); + } + return result; +} + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/Timing.h b/backends/vulkan/test/ops/benchmark_v2/framework/Timing.h new file mode 100644 index 00000000000..8d69d8344c4 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/Timing.h @@ -0,0 +1,23 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include + +#include +#include + +namespace vkcompute::shader_profiling { + +Report measure( + const Json& spec, + ComputeGraph& graph, + const std::function& encode_batch, + const Json& work_spec = Json::object()); + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/Validation.cpp b/backends/vulkan/test/ops/benchmark_v2/framework/Validation.cpp new file mode 100644 index 00000000000..67ff428f542 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/Validation.cpp @@ -0,0 +1,148 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include +#include + +namespace vkcompute::shader_profiling { + +void copy_validation_inputs( + const Json& spec, + ComputeGraph& graph, + const std::map& tensors, + const std::map>& host_data) { + for (const auto& tensor : tensors) { + const Json& description = spec.at("tensors").at(tensor.first); + if (description.value("storage", "buffer") == "constant") { + continue; + } + auto dtype = scalar_type(description.value("dtype", "float32")); + if (dtype == vkapi::kHalf) { + dtype = vkapi::kFloat; + } else if (dtype == vkapi::kInt8x4) { + dtype = vkapi::kChar; + } + graph.maybe_cast_and_copy_into_staging( + graph.staging_of(tensor.second), + host_data.at(tensor.first).data(), + tensor_numel(description.at("sizes")), + dtype); + } +} + +std::map add_validation_outputs( + const Json& spec, + ComputeGraph& graph, + const std::map& tensors, + const Json& accesses) { + std::set outputs = validation_output_names(spec); + const bool explicitly_selected = !outputs.empty(); + if (outputs.empty() && spec.contains("outputs")) { + for (const Json& name : spec.at("outputs")) { + outputs.insert(name.get()); + } + } else if (outputs.empty()) { + for (const auto& access : accesses.items()) { + if (access.value() != "read") { + outputs.insert(access.key()); + } + } + } + if (outputs.empty()) { + throw std::invalid_argument( + "Validation requires at least one output tensor"); + } + std::map staging; + for (const auto& name : outputs) { + if (explicitly_selected) { + reference::require( + accesses.contains(name) && accesses.at(name) != "read", + "Selected validation tensor is not writable: " + name); + } + const ValueRef tensor = tensors.at(name); + auto dtype = graph.dtype_of(tensor); + if (dtype == vkapi::kHalf) { + dtype = vkapi::kFloat; + } else if (dtype == vkapi::kInt8x4) { + dtype = vkapi::kChar; + } + staging.emplace(name, graph.set_output_tensor(tensor, dtype)); + } + return staging; +} + +namespace { + +template +std::vector read_staging( + ComputeGraph& graph, + ValueRef staging, + size_t count, + vkapi::ScalarType dtype) { + std::vector data(count); + graph.maybe_cast_and_copy_from_staging(staging, data.data(), count, dtype); + return std::vector(data.begin(), data.end()); +} + +} // namespace + +Report validate_outputs( + const Json& spec, + ComputeGraph& graph, + const std::map& tensors, + const std::map& staging, + const std::map>& host_data) { + ReferenceOutputs actual; + for (const auto& output : staging) { + const ValueRef tensor = tensors.at(output.first); + if (graph.sizes_of(tensor) != + spec.at("tensors") + .at(output.first) + .at("sizes") + .get>()) { + throw std::invalid_argument( + "Output tensor shape does not match declared shape: " + output.first); + } + const size_t count = tensor_numel(graph.sizes_of(tensor)); + const auto dtype = graph.dtype_of(tensor); + if (dtype == vkapi::kChar || dtype == vkapi::kInt8x4) { + actual[output.first] = + read_staging(graph, output.second, count, vkapi::kChar); + } else if (dtype == vkapi::kByte) { + actual[output.first] = + read_staging(graph, output.second, count, vkapi::kByte); + } else if (dtype == vkapi::kInt) { + actual[output.first] = + read_staging(graph, output.second, count, vkapi::kInt); + } else { + actual[output.first] = + read_staging(graph, output.second, count, vkapi::kFloat); + } + } + const Json dtypes = actual_tensor_dtypes(graph, tensors); + const auto start = std::chrono::steady_clock::now(); + Json reference_spec = spec; + Json shaders = Json::array(); + for (const auto& node : graph.execute_nodes()) { + shaders.push_back(node->name()); + } + reference_spec["reference_context"] = { + {"int8_dot_product", graph.can_use_int8_dot_product()}, + {"shaders", shaders}}; + const ReferenceOutputs expected = + reference_outputs(reference_spec, dtypes, host_data); + const double reference_us = std::chrono::duration( + std::chrono::steady_clock::now() - start) + .count(); + Report result = compare_outputs(spec, expected, actual, dtypes); + result["reference_wall_time_us"] = reference_us; + return result; +} + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/Validation.h b/backends/vulkan/test/ops/benchmark_v2/framework/Validation.h new file mode 100644 index 00000000000..15b91f33e79 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/Validation.h @@ -0,0 +1,33 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include + +namespace vkcompute::shader_profiling { + +void copy_validation_inputs( + const Json& spec, + ComputeGraph& graph, + const std::map& tensors, + const std::map>& host_data); + +std::map add_validation_outputs( + const Json& spec, + ComputeGraph& graph, + const std::map& tensors, + const Json& accesses); + +Report validate_outputs( + const Json& spec, + ComputeGraph& graph, + const std::map& tensors, + const std::map& staging, + const std::map>& host_data); + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/WorkgroupUtils.cpp b/backends/vulkan/test/ops/benchmark_v2/framework/WorkgroupUtils.cpp new file mode 100644 index 00000000000..5f4227cd0bb --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/WorkgroupUtils.cpp @@ -0,0 +1,110 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include + +#include +#include + +namespace vkcompute::shader_profiling { +namespace { + +GlobalWorkGrid pick_gwg( + const Json& setting, + ComputeGraph& graph, + const vkapi::ShaderInfo& shader, + const std::vector& args) { + if (setting.is_array()) { + const auto dims = setting.get>(); + return GlobalWorkGrid({dims[0], dims[1], dims[2]}, kExplicitWorkGrid); + } + static const std::map pickers = { + {"default_pick_gwg", default_pick_gwg}}; + const std::string name = setting.get(); + const auto picker = pickers.find(name); + if (picker == pickers.end()) { + throw std::invalid_argument("Unknown gwg_size picker: " + name); + } + if (args.empty() || !(args.front().access & vkapi::WRITE)) { + throw std::invalid_argument( + "gwg_size picker requires a write or read_write tensor binding"); + } + return picker->second(&graph, shader, args, {}); +} + +LocalWorkGroup pick_lwg( + const Json& setting, + ComputeGraph& graph, + const vkapi::ShaderInfo& shader, + const GlobalWorkGrid& gwg, + const std::vector& args) { + if (setting.is_array()) { + const auto dims = setting.get>(); + return LocalWorkGroup(dims[0], dims[1], dims[2]); + } + static const std::map pickers = { + {"default_pick_lwg", default_pick_lwg}, + {"pick_required_lwg", pick_required_lwg}, + {"pick_xy_square_lwg", pick_xy_square_lwg}, + {"pick_xz_square_lwg", pick_xz_square_lwg}, + {"pick_120shape_lwg", pick_120shape_lwg}}; + const std::string name = setting.get(); + const auto picker = pickers.find(name); + if (picker == pickers.end()) { + throw std::invalid_argument("Unknown lwg_size picker: " + name); + } + return picker->second(&graph, shader, gwg, args, {}); +} + +} // namespace + +WorkgroupSizes pick_workgroup_sizes( + const Json& spec, + ComputeGraph& graph, + const vkapi::ShaderInfo& shader, + const std::map& tensors) { + std::vector args; + std::vector read_args; + for (const Json& binding : spec.at("bindings")) { + if (!binding.contains("tensor")) { + continue; + } + const ValueRef ref = tensors.at(binding.at("tensor").get()); + const std::string access = binding.value("access", "read"); + if (access == "read") { + read_args.emplace_back(ref, vkapi::kRead); + } else { + args.emplace_back( + ref, access == "write" ? vkapi::kWrite : vkapi::kReadWrite); + } + } + for (const ArgGroup& arg : read_args) { + args.emplace_back(arg.refs, arg.access); + } + GlobalWorkGrid gwg = pick_gwg(spec.at("gwg_size"), graph, shader, args); + const LocalWorkGroup lwg = + pick_lwg(spec.at("lwg_size"), graph, shader, gwg, args); + auto* adapter = graph.context()->adapter_ptr(); + lwg.validate( + adapter->max_compute_workgroup_size(), + adapter->max_compute_workgroup_invocations()); + if (gwg.is_linear() && gwg.required_lwg_size() != lwg) { + // Rewrap the linear workload using the selected local geometry. + gwg = GlobalWorkGrid( + {utils::safe_downcast( + graph.numel_of(args.front().refs.front())), + 1, + 1}, + kLinearWorkGrid); + gwg.wrap_linear_dispatch(adapter->max_compute_workgroup_count(), lwg); + } + gwg.validate(lwg, adapter->max_compute_workgroup_count()); + return {gwg, lwg}; +} + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/WorkgroupUtils.h b/backends/vulkan/test/ops/benchmark_v2/framework/WorkgroupUtils.h new file mode 100644 index 00000000000..ef76c607641 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/WorkgroupUtils.h @@ -0,0 +1,28 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include + +#include +#include + +namespace vkcompute::shader_profiling { + +struct WorkgroupSizes { + GlobalWorkGrid gwg; + LocalWorkGroup lwg; +}; + +WorkgroupSizes pick_workgroup_sizes( + const Json& spec, + ComputeGraph& graph, + const vkapi::ShaderInfo& shader, + const std::map& tensors); + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/test/case_check_registry_test.cpp b/backends/vulkan/test/ops/benchmark_v2/framework/test/case_check_registry_test.cpp new file mode 100644 index 00000000000..f54c2ec12f3 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/test/case_check_registry_test.cpp @@ -0,0 +1,92 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include +#include +#include +namespace vkcompute::shader_profiling { +using GpuCaseCheckRegistry = VulkanTest; + +namespace { +Json pass_check(const Json&, ComputeGraph&) { + return {{"status", "PASS"}}; +} +Json fail_check(const Json&, ComputeGraph&) { + return {{"status", "FAIL"}, {"error", "intentional"}}; +} +Json unsupported_shader_check(const Json&, ComputeGraph&) { + throw vkapi::ShaderNotSupportedError( + "unsupported_test_shader", vkapi::VulkanExtension::INTEGER_DOT_PRODUCT); +} +Json unsupported_route_check(const Json&, ComputeGraph&) { + throw UnsupportedRoute("Unsupported test route"); +} +Json error_check(const Json&, ComputeGraph&) { + throw std::runtime_error("Ordinary test error"); +} +REGISTER_CASE_CHECKS { + REGISTER_CASE_CHECK(test_case_check_pass, pass_check); + REGISTER_CASE_CHECK(test_case_check_fail, fail_check); + REGISTER_CASE_CHECK( + test_case_check_unsupported_shader, unsupported_shader_check); + REGISTER_CASE_CHECK( + test_case_check_unsupported_route, unsupported_route_check); + REGISTER_CASE_CHECK(test_case_check_error, error_check); +} +TEST(CaseCheckRegistry, RejectsNullAndDuplicateChecks) { + CaseCheckRegistry registry; + registry.register_check("check", pass_check); + EXPECT_EQ(registry.get_check("check"), pass_check); + EXPECT_THROW( + registry.register_check("check", fail_check), std::invalid_argument); + EXPECT_EQ(registry.get_check("check"), pass_check); + EXPECT_THROW(registry.register_check("", pass_check), std::invalid_argument); + EXPECT_THROW(registry.register_check("null", nullptr), std::invalid_argument); + EXPECT_THROW(registry.get_check("missing"), std::invalid_argument); +} +TEST_F(GpuCaseCheckRegistry, ExecutesRegisteredChecksAndSurfacesFailure) { + const auto pass = run_case_check( + {{"name", "pass"}, + {"check", "test_case_check_pass"}, + {"legacy_case", {{"custom", true}}}}); + EXPECT_EQ(pass.at("validation").at("status"), "PASS"); + EXPECT_FALSE(pass.contains("legacy_case")); + const auto fail = run_case_check({{"check", "test_case_check_fail"}}); + EXPECT_EQ(fail.at("validation").at("status"), "FAIL"); + EXPECT_EQ(fail.at("validation").at("error"), "intentional"); +} +TEST(CaseCheckRegistry, ParsesOnlyUnambiguousStandaloneChecks) { + const auto parsed = parse_configs( + {{"check", "test_case_check_pass"}, {"parameters", {{"value", 3}}}}); + ASSERT_EQ(parsed.size(), 1); + EXPECT_EQ(parsed[0].at("parameters").at("value"), 3); + EXPECT_THROW(parse_configs({{"check", ""}}), std::invalid_argument); + EXPECT_THROW( + parse_configs({{"check", "x"}, {"operator", "op"}}), + std::invalid_argument); + EXPECT_THROW( + parse_configs({{"check", "x"}, {"args_sweep", Json::array()}}), + std::invalid_argument); +} +TEST_F(GpuCaseCheckRegistry, SkipsNotSupportedExceptionsWithoutRequirements) { + for (const char* name : + {"test_case_check_unsupported_shader", + "test_case_check_unsupported_route"}) { + const auto result = run_case_check({{"check", name}}); + EXPECT_EQ(result.at("status"), "SKIP"); + EXPECT_EQ(result.at("validation").at("status"), "SKIP"); + EXPECT_FALSE(result.at("skip_reason").get().empty()); + } +} +TEST_F(GpuCaseCheckRegistry, OrdinaryExceptionsRemainFailures) { + const auto result = run_case_check({{"check", "test_case_check_error"}}); + EXPECT_EQ(result.at("validation").at("status"), "FAIL"); + EXPECT_EQ(result.at("validation").at("error"), "Ordinary test error"); + EXPECT_FALSE(result.contains("skip_reason")); +} +} // namespace +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/test/config_test.cpp b/backends/vulkan/test/ops/benchmark_v2/framework/test/config_test.cpp new file mode 100644 index 00000000000..0568a50abeb --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/test/config_test.cpp @@ -0,0 +1,937 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include + +#include +#include +#include + +namespace vkcompute::shader_profiling { +namespace { + +Json config() { + return { + {"shader", "buffer_to_buffer_float"}, + {"gwg_size", {8, 1, 1}}, + {"lwg_size", {8, 1, 1}}, + {"bindings", + {{{"buffer", {{"name", "out"}, {"sizes", {8}}, {"access", "write"}}}}}}}; +} + +Json operator_config() { + const Json args = Json::array( + {{{"tensor", {{"name", "x"}, {"sizes", {2, 8}}}}}, + {{"tensor", {{"name", "y"}, {"sizes", {2, 4}}}}}}); + return { + {"operator", "aten.linear.default"}, {"args_sweep", Json::array({args})}}; +} + +TEST(ShaderProfilingConfig, AcceptsSingleAndMultipleConfigs) { + EXPECT_EQ(parse_configs(config()).size(), 1); + EXPECT_EQ(parse_configs(Json::array({config(), config()})).size(), 2); + EXPECT_THROW(parse_configs(Json::array()), std::invalid_argument); +} + +TEST(ShaderProfilingConfig, ThroughputFlagsMustBeBooleans) { + for (const char* key : + {"show_memory_throughput", "show_compute_throughput"}) { + Json spec = config(); + spec[key] = true; + EXPECT_EQ(parse_configs(spec)[0].at(key), true); + spec[key] = false; + EXPECT_EQ(parse_configs(spec)[0].at(key), false); + for (const Json& invalid : {Json("true"), Json(1), Json(), Json::array()}) { + spec[key] = invalid; + EXPECT_THROW(parse_configs(spec), std::invalid_argument); + } + } +} + +TEST(ShaderProfilingConfig, ExpandsShaderNamesWithSharedIndependentInputs) { + Json spec = config(); + spec.erase("shader"); + spec["shader_sweep"] = {"shader_b", "shader_a", "shader_b"}; + spec["name"] = "shared inputs"; + spec["target_ms"] = 7; + spec["reference"] = "copy"; + auto parsed = parse_configs(spec); + ASSERT_EQ(parsed.size(), 3); + for (size_t i = 0; i < parsed.size(); ++i) { + EXPECT_EQ(parsed[i].at("shader"), spec.at("shader_sweep")[i]); + EXPECT_EQ(parsed[i].at("sweep_index"), i); + EXPECT_EQ(parsed[i].at("name"), "shared inputs"); + EXPECT_EQ(parsed[i].at("target_ms"), 7); + EXPECT_EQ(parsed[i].at("reference"), "copy"); + EXPECT_FALSE(parsed[i].contains("shader_sweep")); + EXPECT_EQ(parsed[i].at("bindings"), parsed[0].at("bindings")); + EXPECT_EQ(parsed[i].at("tensors"), parsed[0].at("tensors")); + } + parsed[0]["tensors"]["out"]["sizes"] = {16}; + EXPECT_EQ(parsed[1].at("tensors").at("out").at("sizes"), Json::array({8})); +} + +TEST(ShaderProfilingConfig, CombinesShaderAndLocalWorkgroupListsInOrder) { + Json spec = config(); + spec.erase("shader"); + spec["shader_sweep"] = {"shader_a", "shader_b"}; + spec["lwg_size"] = {{8, 1, 1}, {10, 2, 1}}; + const auto parsed = parse_configs(spec); + ASSERT_EQ(parsed.size(), 4); + for (size_t i = 0; i < parsed.size(); ++i) { + EXPECT_EQ(parsed[i].at("shader"), spec.at("shader_sweep")[i / 2]); + EXPECT_EQ(parsed[i].at("lwg_size"), spec.at("lwg_size")[i % 2]); + EXPECT_EQ(parsed[i].at("sweep_index"), i); + } +} + +TEST(ShaderProfilingConfig, CombinesShaderListAndCartesianWorkgroupRange) { + Json spec = config(); + spec.erase("shader"); + spec["shader_sweep"] = {"shader_a", "shader_b"}; + spec.erase("lwg_size"); + spec["lwg_size_sweep"] = {{"x", {8, 12, 4}}, {"y", {1, 2, 1}}}; + const auto parsed = parse_configs(spec); + ASSERT_EQ(parsed.size(), 8); + const Json sizes = {{8, 1, 1}, {8, 2, 1}, {12, 1, 1}, {12, 2, 1}}; + for (size_t i = 0; i < parsed.size(); ++i) { + EXPECT_EQ(parsed[i].at("shader"), spec.at("shader_sweep")[i / 4]); + EXPECT_EQ(parsed[i].at("lwg_size"), sizes[i % 4]); + EXPECT_EQ(parsed[i].at("sweep_index"), i); + EXPECT_FALSE(parsed[i].contains("lwg_size_sweep")); + } +} + +TEST(ShaderProfilingConfig, RejectsMalformedOrAmbiguousShaderSweeps) { + Json spec = config(); + spec["shader_sweep"] = {"shader_a"}; + EXPECT_THROW(parse_configs(spec), std::invalid_argument); + Json op = operator_config(); + op["shader_sweep"] = {"shader_a"}; + EXPECT_THROW(parse_configs(op), std::invalid_argument); + spec.erase("shader"); + for (const Json& invalid : + {Json::array(), + Json("shader_a"), + Json::object(), + Json::array({""}), + Json::array({"shader_a", 1}), + Json::array({nullptr})}) { + spec["shader_sweep"] = invalid; + EXPECT_THROW(parse_configs(spec), std::invalid_argument); + } +} + +TEST(ShaderProfilingConfig, CapsCombinedShaderAndWorkgroupSweep) { + Json spec = config(); + spec.erase("shader"); + spec["shader_sweep"] = {"shader_a", "shader_b"}; + spec.erase("lwg_size"); + spec["lwg_size_sweep"] = {{"x", {1, 32769, 1}}}; + EXPECT_THROW(parse_configs(spec), std::invalid_argument); + spec["shader_sweep"] = std::vector(65537, "shader_a"); + spec.erase("lwg_size_sweep"); + spec["lwg_size"] = {8, 1, 1}; + EXPECT_THROW(parse_configs(spec), std::invalid_argument); +} + +TEST(ShaderProfilingConfig, PreservesMixedDefinitionOrderWithShaderSweeps) { + Json sweep = config(); + sweep.erase("shader"); + sweep["shader_sweep"] = {"shader_a", "shader_b"}; + const auto parsed = parse_configs( + Json::array({config(), sweep, operator_config(), config()})); + ASSERT_EQ(parsed.size(), 5); + EXPECT_EQ(parsed[0].at("shader"), "buffer_to_buffer_float"); + EXPECT_EQ(parsed[1].at("shader"), "shader_a"); + EXPECT_EQ(parsed[2].at("shader"), "shader_b"); + EXPECT_TRUE(parsed[3].contains("operator")); + EXPECT_EQ(parsed[4].at("shader"), "buffer_to_buffer_float"); + EXPECT_FALSE(parsed[0].contains("sweep_index")); + EXPECT_FALSE(parsed[4].contains("sweep_index")); +} + +TEST(ShaderProfilingConfig, RejectsInvalidGridAndTensorSizes) { + Json spec = config(); + for (const Json& invalid : + {Json{0, 1, 1}, Json{-1, 1, 1}, Json{1.5, 1, 1}, Json{1, 1}}) { + spec["gwg_size"] = invalid; + EXPECT_THROW(parse_configs(spec), std::invalid_argument); + } + EXPECT_EQ(tensor_numel({2, 3, 4}), 24); + EXPECT_THROW(tensor_numel({2, 0}), std::invalid_argument); + EXPECT_THROW(tensor_numel({2147483647, 2}), std::invalid_argument); +} + +TEST(ShaderProfilingConfig, RejectsUnknownTensorAndInvalidOptions) { + Json spec = config(); + spec["bindings"][0] = {{"buffer", {{"ref", "missing"}}}}; + EXPECT_THROW(parse_configs(spec), std::invalid_argument); + spec = config(); + spec["benchmark_runs"] = 0; + EXPECT_THROW(parse_configs(spec), std::invalid_argument); +} + +TEST(ShaderProfilingConfig, PacksTypedInputsAndRejectsTruncation) { + auto bytes = make_data({{"dtype", "int32"}, {"data", {-7, 42}}}, 2); + int32_t values[2]; + std::memcpy(values, bytes.data(), sizeof(values)); + EXPECT_EQ(values[0], -7); + EXPECT_EQ(values[1], 42); + EXPECT_THROW( + make_data({{"dtype", "int8"}, {"data", {128}}}, 1), + std::invalid_argument); + EXPECT_THROW(make_data({{"data", {1, 2}}}, 3), std::invalid_argument); +} + +TEST(ShaderProfilingConfig, StructuredDataInitializersAreTypedAndReproducible) { + const auto constants = + make_data({{"dtype", "int8"}, {"data", {{"fill", -7}}}}, 4); + EXPECT_EQ(constants, std::vector(4, 249)); + const Json spec = { + {"dtype", "float32"}, {"data", {{"rand_uniform_float", {0, 1}}}}}; + const auto random = make_data(spec, 64, 7); + EXPECT_EQ(random, make_data(spec, 64, 7)); + EXPECT_NE(random, make_data(spec, 64, 8)); + Json half = spec; + half["dtype"] = "float16"; + EXPECT_EQ(random, make_data(half, 64, 7)); + float values[64]; + std::memcpy(values, random.data(), sizeof(values)); + for (float value : values) { + EXPECT_GE(value, 0); + EXPECT_LE(value, 1); + } + Json seeded = spec; + seeded["data"]["seed"] = 42; + EXPECT_EQ(make_data(seeded, 64, 1), make_data(seeded, 64, 2)); +} + +TEST(ShaderProfilingConfig, UniformIntegerDataIsInclusiveAndReproducible) { + const Json spec = { + {"dtype", "int8"}, {"data", {{"rand_uniform_int", {-4, 4}}}}}; + const auto data = make_data(spec, 512, 7); + EXPECT_EQ(data, make_data(spec, 512, 7)); + EXPECT_NE(data, make_data(spec, 512, 8)); + int8_t values[512]; + std::memcpy(values, data.data(), sizeof(values)); + bool saw_min = false; + bool saw_max = false; + for (int8_t value : values) { + EXPECT_GE(value, -4); + EXPECT_LE(value, 4); + saw_min |= value == -4; + saw_max |= value == 4; + } + EXPECT_TRUE(saw_min); + EXPECT_TRUE(saw_max); + const auto large = make_data( + {{"dtype", "uint32"}, + {"data", {{"rand_uniform_int", {0, 2147483648LL}}}}}, + 256); + uint32_t wide[256]; + std::memcpy(wide, large.data(), sizeof(wide)); + for (uint32_t value : wide) { + EXPECT_LE(value, 2147483648u); + } + EXPECT_THROW( + make_data({{"data", {{"rand_uniform_int", {0, 1.5}}}}}, 4), + std::invalid_argument); + EXPECT_THROW( + make_data( + {{"dtype", "int8"}, {"data", {{"rand_uniform_int", {-129, 127}}}}}, + 4), + std::invalid_argument); + EXPECT_THROW( + make_data( + {{"dtype", "int32"}, {"data", {{"rand_uniform_int", {0, 1.5}}}}}, 4), + std::invalid_argument); +} + +TEST( + ShaderProfilingConfig, + RowSumsTrackRandomWeightsAndResolveForwardReferences) { + const Json tensors = { + {"a_sums", + {{"sizes", {2}}, + {"dtype", "int32"}, + {"data", {{"row_sum", "z_weights"}}}}}, + {"z_weights", + {{"sizes", {2, 3}}, + {"dtype", "int8"}, + {"data", {{"rand_uniform_int", {-4, 4}}}}}}}; + const auto data = make_tensor_data(tensors); + int8_t weights[6]; + int32_t sums[2]; + std::memcpy(weights, data.at("z_weights").data(), sizeof(weights)); + std::memcpy(sums, data.at("a_sums").data(), sizeof(sums)); + EXPECT_EQ(sums[0], weights[0] + weights[1] + weights[2]); + EXPECT_EQ(sums[1], weights[3] + weights[4] + weights[5]); + EXPECT_EQ(data, make_tensor_data(tensors)); + Json invalid = tensors; + invalid["a_sums"]["data"]["row_sum"] = "missing"; + EXPECT_THROW(make_tensor_data(invalid), std::invalid_argument); + invalid["a_sums"]["data"]["row_sum"] = "a_sums"; + EXPECT_THROW(make_tensor_data(invalid), std::invalid_argument); + invalid = tensors; + invalid["a_sums"]["sizes"] = {1}; + EXPECT_THROW(make_tensor_data(invalid), std::invalid_argument); +} + +TEST(ShaderProfilingConfig, RejectsMalformedDataInitializers) { + for (const Json& data : + {Json::object(), + Json{{"unknown", 0}}, + Json{{"constant", 0}}, + Json{{"fill", "bad"}}, + Json{{"rand_uniform_float", {1, 0}}}, + Json{{"rand_uniform_float", {0}}}, + Json{{"rand_uniform_float", {0, 1}}, {"seed", -1}}, + Json{{"fill", 0}, {"rand_uniform_float", {0, 1}}}}) { + EXPECT_THROW(make_data({{"data", data}}, 4), std::invalid_argument); + } + EXPECT_THROW( + make_data( + {{"dtype", "int8"}, {"data", {{"rand_uniform_float", {0, 1}}}}}, 4), + std::invalid_argument); + EXPECT_THROW( + make_data({{"dtype", "int8"}, {"data", {{"fill", 128}}}}, 4), + std::invalid_argument); + EXPECT_THROW( + make_data({{"fill", 1}, {"data", {{"fill", 0}}}}, 4), + std::invalid_argument); +} + +TEST(ShaderProfilingConfig, BoundsAdaptiveDispatchCount) { + EXPECT_EQ(choose_dispatch_count(10, 1000, 4096), 100); + EXPECT_EQ(choose_dispatch_count(2000, 1000, 4096), 1); + EXPECT_EQ(choose_dispatch_count(0, 1000, 128), 128); +} + +TEST(ShaderProfilingConfig, AcceptsOperatorAndMixedCases) { + Json op = operator_config(); + op["args_sweep"][0].push_back(nullptr); + op["args_sweep"][0].push_back({{"int_list", {1, 2}}}); + EXPECT_EQ(parse_configs(op).size(), 1); + EXPECT_EQ(parse_configs(Json::array({config(), op})).size(), 2); +} + +TEST(ShaderProfilingConfig, ConstantStorageIsOperatorOnlyAndReplacesTheFlag) { + Json op = operator_config(); + op["args_sweep"][0][0]["tensor"]["storage"] = "constant"; + op["args_sweep"][0][0]["tensor"]["sizes"] = {1, 2, 3, 4, 5}; + const Json parsed = parse_configs(op).at(0); + EXPECT_EQ(parsed.at("tensors").at("x").at("storage"), "constant"); + EXPECT_FALSE(parsed.at("tensors").at("x").contains("constant")); + op = operator_config(); + op["args_sweep"][0][0]["tensor"]["constant"] = true; + EXPECT_THROW(parse_configs(op), std::invalid_argument); + Json shader = config(); + shader["bindings"][0]["buffer"]["storage"] = "constant"; + EXPECT_THROW(parse_configs(shader), std::invalid_argument); +} + +TEST(ShaderProfilingConfig, CartesianVariablesPreserveTypesAndTemplateOrder) { + Json op = operator_config(); + op["sweep_variables"] = { + {"v_DTYPE", {"float32", "float16"}}, + {"v_STORAGE", {"buffer", "texture3d"}}, + {"v_M", {4}}}; + for (auto& arg : op["args_sweep"][0]) { + arg["tensor"]["dtype"] = "v_DTYPE"; + arg["tensor"]["storage"] = "v_STORAGE"; + arg["tensor"]["sizes"][0] = "v_M"; + arg["tensor"]["data"] = {{"rand_uniform_float", {0, 1}}}; + } + op["args_sweep"].push_back(op["args_sweep"][0]); + const auto parsed = parse_configs(op); + ASSERT_EQ(parsed.size(), 8); + for (size_t i = 0; i < parsed.size(); ++i) { + EXPECT_EQ(parsed[i].at("sweep_index"), i); + EXPECT_EQ(parsed[i].at("args_sweep_index"), i / 4); + EXPECT_EQ(parsed[i].at("tensors").at("x").at("sizes")[0], 4); + EXPECT_EQ( + parsed[i].at("tensors").at("x").at("dtype"), + i % 4 < 2 ? "float32" : "float16"); + EXPECT_EQ( + parsed[i].at("tensors").at("x").at("storage"), + i % 2 == 0 ? "buffer" : "texture3d"); + EXPECT_EQ(parsed[i].at("sweep_variables").at("v_M"), 4); + EXPECT_FALSE(parsed[i].at("args")[1].contains("access")); + } +} + +TEST(ShaderProfilingConfig, RejectsInvalidOrUnboundSweepVariables) { + for (const Json& variables : + {Json::array(), + Json{{"v_DTYPE", Json::array()}}, + Json{{"v_DTYPE", "float32"}}, + Json{{"DTYPE", {"float32"}}}}) { + Json op = operator_config(); + op["sweep_variables"] = variables; + EXPECT_THROW(parse_configs(op), std::invalid_argument); + } + Json op = operator_config(); + op["args_sweep"][0][0]["tensor"]["dtype"] = "v_MISSING"; + EXPECT_THROW(parse_configs(op), std::invalid_argument); +} + +TEST(ShaderProfilingConfig, NormalizesExplicitOperatorRoutes) { + Json op = operator_config(); + EXPECT_FALSE(parse_configs(op).at(0).contains("route")); + op["route"] = "direct"; + EXPECT_EQ( + parse_configs(op).at(0).at("route"), + (Json{{"name", "direct"}, {"options", Json::object()}})); + op["route"] = {{"name", "tiled"}}; + EXPECT_EQ( + parse_configs(op).at(0).at("route"), + (Json{{"name", "tiled"}, {"options", Json::object()}})); + op["route"]["options"] = {{"tile", {2, 4}}, {"custom_option", true}}; + EXPECT_EQ(parse_configs(op).at(0).at("route"), op.at("route")); +} + +TEST(ShaderProfilingConfig, ExpandsRouteSweepInsideArgumentVariables) { + Json op = operator_config(); + op["sweep_variables"] = {{"v_DTYPE", {"float32", "float16"}}}; + op["args_sweep"][0][0]["tensor"]["dtype"] = "v_DTYPE"; + op["args_sweep"][0][0]["tensor"]["data"] = {{"rand_uniform_float", {-1, 1}}}; + op["args_sweep"].push_back(op["args_sweep"][0]); + op["args_sweep"][1][0]["tensor"]["sizes"] = {4, 8}; + op["route_sweep"] = Json::array( + {"auto", {{"name", "tiled"}, {"options", {{"tile_rows", 2}}}}}); + auto cases = parse_configs(op); + ASSERT_EQ(cases.size(), 8); + for (size_t i = 0; i < cases.size(); ++i) { + EXPECT_EQ(cases[i].at("sweep_index"), i); + EXPECT_EQ(cases[i].at("args_sweep_index"), i / 4); + EXPECT_EQ( + cases[i].at("sweep_variables").at("v_DTYPE"), + i % 4 < 2 ? "float32" : "float16"); + EXPECT_EQ(cases[i].at("route").at("name"), i % 2 == 0 ? "auto" : "tiled"); + EXPECT_FALSE(cases[i].contains("route_sweep")); + EXPECT_EQ(cases[i].at("tensors").at("x").at("sizes")[0], i < 4 ? 2 : 4); + } + EXPECT_EQ( + make_tensor_data(cases[0].at("tensors")), + make_tensor_data(cases[1].at("tensors"))); + cases[0]["tensors"]["x"]["sizes"] = {8, 8}; + EXPECT_EQ(cases[1].at("tensors").at("x").at("sizes"), (Json{2, 8})); +} + +TEST(ShaderProfilingConfig, SubstitutesRouteNamesAndTypedOptions) { + Json op = operator_config(); + op["sweep_variables"] = { + {"v_ROUTE", {"direct", "tiled"}}, {"v_TILE", {{2, 4}, {4, 8}}}}; + op["route"] = {{"name", "v_ROUTE"}, {"options", {{"tile", "v_TILE"}}}}; + const auto cases = parse_configs(op); + ASSERT_EQ(cases.size(), 4); + EXPECT_EQ(cases[0].at("route").at("name"), "direct"); + EXPECT_EQ(cases[2].at("route").at("name"), "tiled"); + EXPECT_EQ(cases[0].at("route").at("options").at("tile"), (Json{2, 4})); + EXPECT_EQ(cases[1].at("route").at("options").at("tile"), (Json{4, 8})); + op.erase("route"); + op["route_sweep"] = + Json::array({"v_ROUTE", {{"name", "auto"}, {"options", "v_OPTIONS"}}}); + op["sweep_variables"] = { + {"v_ROUTE", {"direct", "tiled"}}, + {"v_OPTIONS", Json::array({Json{{"tile_rows", 2}}})}}; + const auto swept = parse_configs(op); + ASSERT_EQ(swept.size(), 4); + EXPECT_EQ(swept[0].at("route").at("name"), "direct"); + EXPECT_EQ(swept[2].at("route").at("name"), "tiled"); + EXPECT_EQ(swept[1].at("route").at("options").at("tile_rows"), 2); +} + +TEST(ShaderProfilingConfig, RejectsMalformedOrShaderRouteSelections) { + for (const Json& invalid : + {Json(nullptr), + Json(1), + Json(false), + Json(""), + Json::array({"direct"}), + Json::object(), + Json{{"options", Json::object()}}, + Json{{"name", 1}}, + Json{{"name", ""}}, + Json{{"name", "direct"}, {"unknown", 1}}, + Json{{"name", "direct"}, {"options", nullptr}}, + Json{{"name", "direct"}, {"options", Json::array()}}, + Json{{"name", "direct"}, {"options", 2}}}) { + Json op = operator_config(); + op["route"] = invalid; + EXPECT_THROW(parse_configs(op), std::invalid_argument); + op.erase("route"); + op["route_sweep"] = Json::array({"auto", invalid}); + EXPECT_THROW(parse_configs(op), std::invalid_argument); + } + for (const Json& invalid : + {Json(nullptr), Json("direct"), Json::object(), Json::array()}) { + Json op = operator_config(); + op["route_sweep"] = invalid; + EXPECT_THROW(parse_configs(op), std::invalid_argument); + } + Json op = operator_config(); + op["route"] = "direct"; + op["route_sweep"] = {"auto"}; + EXPECT_THROW(parse_configs(op), std::invalid_argument); + for (const char* key : {"route", "route_sweep"}) { + Json shader = config(); + shader[key] = + key == std::string("route") ? Json("direct") : Json::array({"direct"}); + EXPECT_THROW(parse_configs(shader), std::invalid_argument); + } + op = operator_config(); + op["route"] = {{"name", "v_MISSING"}}; + EXPECT_THROW(parse_configs(op), std::invalid_argument); + op["sweep_variables"] = {{"v_MISSING", {1}}}; + EXPECT_THROW(parse_configs(op), std::invalid_argument); +} + +TEST(ShaderProfilingConfig, RejectsLegacyRouteSelectionKeys) { + for (const char* key : {"implementation", "implementation_sweep"}) { + for (Json spec : {operator_config(), config()}) { + spec[key] = key == std::string("implementation") + ? Json("direct") + : Json::array({"direct"}); + EXPECT_THROW(parse_configs(spec), std::invalid_argument); + } + } +} + +TEST(ShaderProfilingConfig, CapsCombinedRouteCartesianExpansion) { + Json op = operator_config(); + op["args_sweep"] = Json::array_t(32, Json::array({{{"int", 1}}})); + op["sweep_variables"] = {{"v_VALUE", Json::array_t(128, 1)}}; + op["route_sweep"] = Json::array_t(32, "auto"); + EXPECT_THROW(parse_configs(op), std::invalid_argument); + op["args_sweep"] = Json::array_t(17, Json::array({{{"int", 1}}})); + EXPECT_THROW(parse_configs(op), std::invalid_argument); +} + +TEST( + ShaderProfilingConfig, + ExpandsSweepsInOrderWithIndependentTensorDefinitions) { + Json op = operator_config(); + op["name"] = "linear"; + op["target_ms"] = 7; + Json half_args = op.at("args_sweep").at(0); + half_args[0]["tensor"]["dtype"] = "float16"; + half_args[0]["tensor"]["sizes"] = {4, 8}; + half_args[1]["tensor"]["dtype"] = "float16"; + half_args[1]["tensor"]["sizes"] = {4, 4}; + op["args_sweep"].push_back(half_args); + const auto parsed = parse_configs(Json::array({config(), op, config()})); + ASSERT_EQ(parsed.size(), 4); + EXPECT_TRUE(parsed[0].contains("shader")); + EXPECT_TRUE(parsed[3].contains("shader")); + for (size_t i = 0; i < 2; ++i) { + EXPECT_EQ(parsed[i + 1].at("name"), "linear"); + EXPECT_EQ(parsed[i + 1].at("target_ms"), 7); + EXPECT_EQ(parsed[i + 1].at("sweep_index"), i); + EXPECT_FALSE(parsed[i + 1].contains("args_sweep")); + EXPECT_EQ(parsed[i + 1].at("tensors").size(), 2); + } + EXPECT_EQ(parsed[1].at("tensors").at("x").at("sizes"), Json::array({2, 8})); + EXPECT_EQ(parsed[2].at("tensors").at("x").at("sizes"), Json::array({4, 8})); + EXPECT_EQ( + parsed[1].at("tensors").at("x").value("dtype", "float32"), "float32"); + EXPECT_EQ(parsed[2].at("tensors").at("x").at("dtype"), "float16"); + op["args_sweep"][1][0]["tensor"] = {{"ref", "x"}}; + EXPECT_THROW(parse_configs(op), std::invalid_argument); +} + +TEST(ShaderProfilingConfig, RejectsEmptyAndMalformedSweeps) { + for (const Json& invalid : + {Json::array(), + Json::object(), + Json(1), + Json::array({Json::object()}), + Json::array({Json::array()})}) { + Json op = operator_config(); + op["args_sweep"] = invalid; + EXPECT_THROW(parse_configs(op), std::invalid_argument); + } + Json op = operator_config(); + op.erase("args_sweep"); + EXPECT_THROW(parse_configs(op), std::invalid_argument); + Json shader = config(); + shader["args_sweep"] = Json::array({Json::array()}); + EXPECT_THROW(parse_configs(shader), std::invalid_argument); +} + +TEST(ShaderProfilingConfig, RejectsAmbiguousCasesAndOperatorArguments) { + Json op = operator_config(); + op["shader"] = "buffer_to_buffer_float"; + EXPECT_THROW(parse_configs(op), std::invalid_argument); + op.erase("shader"); + op["args_sweep"][0][0]["tensor"] = {{"ref", "missing"}}; + EXPECT_THROW(parse_configs(op), std::invalid_argument); + op["args_sweep"][0] = Json::array({{{"int", 1.5}}}); + EXPECT_THROW(parse_configs(op), std::invalid_argument); +} + +TEST(ShaderProfilingConfig, SupportsPackedInt8Activations) { + Json op = operator_config(); + op["operator"] = "et_vk.q8ta_conv2d.default"; + op["args_sweep"][0] = Json::array( + {{{"tensor", + {{"sizes", {1, 4, 8, 8}}, {"dtype", "int8x4"}, {"layout", "4w4c"}}}}, + {{"tensor", + {{"sizes", {1, 8, 8, 8}}, + {"dtype", "int8x4"}, + {"layout", "4w4c"}, + {"access", "write"}}}}}); + EXPECT_EQ(parse_configs(op).size(), 1); + EXPECT_EQ( + make_data({{"dtype", "int8x4"}, {"fill", -7}}, 4), + std::vector(4, 249)); +} + +TEST(ShaderProfilingConfig, InlineOperatorTensorsPreserveReferencesAndData) { + Json op = operator_config(); + op["args_sweep"][0] = Json::array( + {{{"tensor", {{"ref", "state"}}}}, + {{"tensor", + {{"name", "weight"}, + {"sizes", {2}}, + {"dtype", "int32"}, + {"storage", "constant"}, + {"data", {-7, 42}}}}}, + nullptr, + {{"tensor", {{"sizes", {2}}, {"fill", 0.5}}}}, + {{"tensor", + {{"name", "state"}, + {"sizes", {2}}, + {"storage", "texture3d"}, + {"access", "write"}}}}, + {{"tensor", {{"ref", "arg_3"}, {"access", "read_write"}}}}}); + const Json parsed = parse_configs(op).at(0); + EXPECT_EQ(parsed.at("tensors").size(), 3); + EXPECT_FALSE(parsed.contains("inputs")); + EXPECT_FALSE(parsed.contains("outputs")); + EXPECT_EQ(parsed.at("args").size(), 6); + EXPECT_EQ(parsed.at("args").at(0).at("tensor"), "state"); + EXPECT_TRUE(parsed.at("args").at(2).is_null()); + EXPECT_EQ(parsed.at("args").at(4).at("tensor"), "state"); + EXPECT_EQ(parsed.at("args").at(4).at("access"), "write"); + EXPECT_EQ(parsed.at("args").at(5).at("tensor"), "arg_3"); + EXPECT_EQ(parsed.at("args").at(5).at("access"), "read_write"); + EXPECT_EQ( + parsed.at("tensors").at("weight").at("data"), Json::array({-7, 42})); + EXPECT_EQ(parsed.at("tensors").at("weight").at("storage"), "constant"); + EXPECT_EQ(parsed.at("tensors").at("arg_3").at("fill"), 0.5); +} + +TEST( + ShaderProfilingConfig, + RejectsExternalOperatorTensorsAndInvalidDefinitions) { + Json op = operator_config(); + op["tensors"] = {{"old", {{"sizes", {8}}}}}; + EXPECT_THROW(parse_configs(op), std::invalid_argument); + for (const Json& invalid : + {Json("x"), + Json{{"name", "x"}}, + Json{{"sizes", {0}}}, + Json{{"name", ""}, {"sizes", {2}}}, + Json{{"name", "y"}, {"sizes", {2}}}, + Json{{"ref", "missing"}}, + Json{{"ref", "y"}, {"sizes", {2}}}, + Json{{"sizes", {2}}, {"dtype", "unsupported"}}}) { + op = operator_config(); + op["args_sweep"][0][0]["tensor"] = invalid; + EXPECT_THROW(parse_configs(op), std::invalid_argument); + } + op = operator_config(); + op["args_sweep"][0][1]["tensor"]["storage"] = "constant"; + op["args_sweep"][0][1]["tensor"]["access"] = "write"; + EXPECT_THROW(parse_configs(op), std::invalid_argument); + op["args_sweep"][0] = Json::array({{{"float", "bad"}}}); + EXPECT_THROW(parse_configs(op), std::invalid_argument); + op["args_sweep"][0] = Json::array(); + EXPECT_THROW(parse_configs(op), std::invalid_argument); +} + +TEST(ShaderProfilingConfig, RejectsOldOperatorListsAndInvalidAccess) { + for (const char* key : {"args", "inputs", "outputs"}) { + Json op = operator_config(); + op[key] = Json::array(); + EXPECT_THROW(parse_configs(op), std::invalid_argument); + } + for (const Json& access : {Json("unknown"), Json(1), Json(false)}) { + Json op = operator_config(); + op["args_sweep"][0][1]["tensor"]["access"] = access; + EXPECT_THROW(parse_configs(op), std::invalid_argument); + } + Json op = operator_config(); + op["args_sweep"][0][0]["tensor"]["storage"] = "constant"; + op["args_sweep"][0][1]["tensor"] = {{"ref", "x"}, {"access", "write"}}; + EXPECT_THROW(parse_configs(op), std::invalid_argument); +} + +TEST(ShaderProfilingConfig, ReadWriteTensorKeepsOneArgument) { + Json op = operator_config(); + op["operator"] = "test.inplace.default"; + op["args_sweep"][0] = + Json::array({{{"tensor", {{"sizes", {8}}, {"access", "read_write"}}}}}); + const Json parsed = parse_configs(op).at(0); + EXPECT_EQ(parsed.at("args").size(), 1); + EXPECT_EQ(parsed.at("args").at(0).at("tensor"), "arg_0"); + EXPECT_EQ(parsed.at("args").at(0).at("access"), "read_write"); +} + +TEST(ShaderProfilingConfig, InlineBindingsPreserveDataAndSharedReferences) { + Json spec = config(); + spec["bindings"] = Json::array( + {{{"meta_ubo", "shared"}, {"field", "numel"}}, + {{"buffer", + {{"name", "shared"}, + {"sizes", {2}}, + {"dtype", "int32"}, + {"data", {-7, 42}}}}}, + {{"buffer", {{"ref", "shared"}, {"access", "read_write"}}}}, + {{"buffer", {{"sizes", {2}}, {"fill", 0.5}}}}}); + const Json parsed = parse_configs(spec).at(0); + EXPECT_EQ(parsed.at("tensors").size(), 2); + EXPECT_EQ( + parsed.at("tensors").at("shared").at("data"), Json::array({-7, 42})); + EXPECT_EQ(parsed.at("tensors").at("shared").at("dtype"), "int32"); + EXPECT_EQ(parsed.at("bindings").at(1).at("tensor"), "shared"); + EXPECT_EQ(parsed.at("bindings").at(2).at("tensor"), "shared"); +} + +TEST( + ShaderProfilingConfig, + RejectsExternalTensorListsAndDuplicateBindingNames) { + Json spec = config(); + spec["tensors"] = {{"old", {{"sizes", {8}}}}}; + EXPECT_THROW(parse_configs(spec), std::invalid_argument); + spec.erase("tensors"); + spec["bindings"].push_back({{"buffer", {{"name", "out"}, {"sizes", {8}}}}}); + EXPECT_THROW(parse_configs(spec), std::invalid_argument); +} + +TEST(ShaderProfilingConfig, RejectsInvalidInlineTensorAndLayoutReferences) { + Json spec = config(); + spec["bindings"][0]["buffer"]["sizes"] = {0}; + EXPECT_THROW(parse_configs(spec), std::invalid_argument); + spec = config(); + spec["specialization_constants"] = Json::array({{{"layout", "missing"}}}); + EXPECT_THROW(parse_configs(spec), std::invalid_argument); + spec = config(); + spec["outputs"] = {"missing"}; + EXPECT_THROW(parse_configs(spec), std::invalid_argument); +} + +TEST(ShaderProfilingConfig, BindingTypeDeterminesTensorStorage) { + for (const char* type : {"buffer", "texture2d", "texture3d"}) { + Json spec = config(); + spec["bindings"] = + Json::array({{{type, {{"name", "input"}, {"sizes", {8}}}}}}); + EXPECT_EQ( + parse_configs(spec).at(0).at("tensors").at("input").at("storage"), + type); + } +} + +TEST(ShaderProfilingConfig, RejectsUntypedAndConflictingBindings) { + Json spec = config(); + spec["bindings"] = Json::array({{{"name", "input"}, {"sizes", {8}}}}); + EXPECT_THROW(parse_configs(spec), std::invalid_argument); + spec = config(); + spec["bindings"][0]["texture3d"] = {{"sizes", {8}}}; + EXPECT_THROW(parse_configs(spec), std::invalid_argument); + spec = config(); + spec["bindings"][0]["buffer"]["storage"] = "texture3d"; + EXPECT_THROW(parse_configs(spec), std::invalid_argument); + spec = config(); + spec["bindings"].push_back({{"texture3d", {{"ref", "out"}}}}); + EXPECT_THROW(parse_configs(spec), std::invalid_argument); +} + +TEST(ShaderProfilingConfig, AcceptsNamedWorkgroupPickers) { + Json spec = config(); + spec["gwg_size"] = "default_pick_gwg"; + spec["lwg_size"] = "default_pick_lwg"; + EXPECT_EQ(parse_configs(spec).at(0).at("gwg_size"), "default_pick_gwg"); + spec["lwg_size"] = {8, 1, 1}; + EXPECT_EQ(parse_configs(spec).size(), 1); + spec["gwg_size"] = {8, 1, 1}; + spec["lwg_size"] = "pick_xy_square_lwg"; + EXPECT_EQ(parse_configs(spec).size(), 1); +} + +TEST(ShaderProfilingConfig, ExpandsLocalWorkgroupListInSpecifiedOrder) { + Json spec = config(); + spec["lwg_size"] = {{8, 8, 1}, {10, 10, 1}}; + const auto cases = parse_configs(spec); + ASSERT_EQ(cases.size(), 2); + EXPECT_EQ(cases[0].at("lwg_size"), (Json{8, 8, 1})); + EXPECT_EQ(cases[1].at("lwg_size"), (Json{10, 10, 1})); + EXPECT_EQ(cases[0].at("sweep_index"), 0); + EXPECT_EQ(cases[1].at("sweep_index"), 1); + EXPECT_EQ(cases[0].at("tensors"), cases[1].at("tensors")); + spec["lwg_size"] = {{8, 1, 1}, {0, 1, 1}}; + EXPECT_THROW(parse_configs(spec), std::invalid_argument); + spec["lwg_size"] = Json::array({Json{8, 1, 1}, "default_pick_lwg"}); + EXPECT_THROW(parse_configs(spec), std::invalid_argument); +} + +TEST( + ShaderProfilingConfig, + ExpandsInclusiveLocalWorkgroupRangeCartesianProduct) { + Json spec = config(); + spec.erase("lwg_size"); + spec["lwg_size_sweep"] = {{"x", {8, 10, 2}}, {"y", {1, 3, 2}}}; + const auto cases = parse_configs(spec); + ASSERT_EQ(cases.size(), 4); + const Json expected = {{8, 1, 1}, {8, 3, 1}, {10, 1, 1}, {10, 3, 1}}; + for (size_t i = 0; i < cases.size(); ++i) { + EXPECT_EQ(cases[i].at("lwg_size"), expected[i]); + EXPECT_EQ(cases[i].at("sweep_index"), i); + EXPECT_FALSE(cases[i].contains("lwg_size_sweep")); + } + spec["lwg_size_sweep"] = {{"x", {8, 11, 2}}}; + EXPECT_EQ(parse_configs(spec).size(), 2); + spec["lwg_size_sweep"] = {{"x", {4294967294ULL, 4294967295ULL, 2}}}; + EXPECT_EQ(parse_configs(spec).size(), 1); +} + +TEST(ShaderProfilingConfig, RejectsInvalidLocalWorkgroupSweeps) { + Json spec = config(); + spec["lwg_size_sweep"] = {{"x", {1, 2, 1}}}; + EXPECT_THROW(parse_configs(spec), std::invalid_argument); + spec.erase("lwg_size"); + for (const Json& invalid : + {Json::array(), + Json{{"w", {1, 2, 1}}}, + Json{{"x", {2, 1, 1}}}, + Json{{"x", {1, 2, 0}}}, + Json{{"x", {1, 2, -1}}}, + Json{{"x", {1, 2}}}, + Json{{"x", {1, 2, 0.5}}}, + Json{{"x", {0, 2, 1}}}, + Json{{"x", {1, 4294967296ULL, 1}}}, + Json{{"x", {1, 65537, 1}}}, + Json{{"x", {1, 257, 1}}, {"y", {1, 256, 1}}}}) { + spec["lwg_size_sweep"] = invalid; + EXPECT_THROW(parse_configs(spec), std::invalid_argument); + } + Json op = operator_config(); + op["lwg_size_sweep"] = {{"x", {1, 2, 1}}}; + EXPECT_THROW(parse_configs(op), std::invalid_argument); +} + +TEST(ShaderProfilingConfig, ValidatesReferenceComparisonOptions) { + Json spec = config(); + spec["reference"] = "copy"; + spec["validation_method"] = { + {"function", "is_close"}, {"atol", 0.01}, {"rtol", 0.02}}; + EXPECT_EQ(parse_configs(spec)[0].at("reference"), "copy"); + spec["reference"] = "custom_reference"; + EXPECT_EQ(parse_configs(spec).size(), 1); + for (const Json& invalid : {Json(0), Json("")}) { + spec["reference"] = invalid; + EXPECT_THROW(parse_configs(spec), std::invalid_argument); + } + spec["reference"] = "copy"; + for (const Json& invalid : + {Json("is_close"), + Json{{"function", "unknown"}}, + Json{{"atol", 0.01}}, + Json{{"function", "is_close"}, {"atol", -1}}, + Json{{"function", "is_close"}, {"rtol", "0.1"}}, + Json{ + {"function", "is_close"}, + {"atol", std::numeric_limits::infinity()}}, + Json{{"function", "is_close"}, {"extra", 1}}}) { + spec["validation_method"] = invalid; + EXPECT_THROW(parse_configs(spec), std::invalid_argument); + } +} + +TEST(ShaderProfilingConfig, SelectsNamedValidationOutputs) { + Json spec = config(); + spec["validation_method"] = {{"function", "is_close"}, {"outputs", {"out"}}}; + EXPECT_EQ( + parse_configs(spec)[0].at("validation_method").at("outputs"), + Json::array({"out"})); + for (const Json& invalid : + {Json("out"), + Json::array(), + Json::array({""}), + Json::array({0}), + Json::array({"missing"}), + Json::array({"out", "out"})}) { + spec["validation_method"]["outputs"] = invalid; + EXPECT_THROW(parse_configs(spec), std::invalid_argument); + } + spec = operator_config(); + spec["validation_method"] = {{"function", "is_close"}, {"outputs", {"y"}}}; + EXPECT_EQ(parse_configs(spec).size(), 1); + spec["args_sweep"][0][1]["tensor"]["storage"] = "constant"; + EXPECT_THROW(parse_configs(spec), std::invalid_argument); +} + +TEST(ShaderProfilingConfig, RejectsInvalidWorkgroupPickerValues) { + for (const char* key : {"gwg_size", "lwg_size"}) { + for (const Json& invalid : + {Json(""), Json(1), Json(false), Json::object()}) { + Json spec = config(); + spec[key] = invalid; + EXPECT_THROW(parse_configs(spec), std::invalid_argument); + } + } +} + +TEST(ShaderProfilingConfig, RejectsOldWorkgroupAndMetadataKeys) { + Json spec = config(); + spec["global_size"] = {8, 1, 1}; + EXPECT_THROW(parse_configs(spec), std::invalid_argument); + spec.erase("global_size"); + spec["local_size"] = {8, 1, 1}; + EXPECT_THROW(parse_configs(spec), std::invalid_argument); + spec = config(); + spec["bindings"].push_back({{"metadata", "out"}}); + EXPECT_THROW(parse_configs(spec), std::invalid_argument); +} + +} // namespace +TEST(ShaderProfilingConfig, RejectsRemovedNarrowInt4Setting) { + for (Json spec : + {config(), + operator_config(), + Json{{"check", "unsigned_dot_selection"}}}) { + for (bool value : {false, true}) { + spec["force_narrow_int4_tile"] = value; + EXPECT_THROW(parse_configs(spec), std::invalid_argument); + } + } +} + +TEST(ShaderProfilingConfig, RejectsRemovedBenchmarkControls) { + const Json removed = { + {"force_resize", true}, + {"invocations_per_batch", 17}, + {"gpu_timing", "batch"}, + {"shader_filter", Json::array()}, + {"requirements", {{"features", {"int8_dot_product"}}}}, + {"device_variables", {{"v_SIZE", {{"property", "subgroup_size"}}}}}}; + for (const Json& base : + {config(), + operator_config(), + Json{{"check", "unsigned_dot_selection"}}}) { + for (const auto& option : removed.items()) { + Json spec = base; + spec[option.key()] = option.value(); + EXPECT_THROW(parse_configs(spec), std::invalid_argument); + } + } +} + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/test/data_initializer_test.cpp b/backends/vulkan/test/ops/benchmark_v2/framework/test/data_initializer_test.cpp new file mode 100644 index 00000000000..ee48ca27e19 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/test/data_initializer_test.cpp @@ -0,0 +1,255 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include +#include +#include +#include +#include + +namespace vkcompute::shader_profiling { +namespace { + +template +std::vector unpack(const std::vector& data) { + std::vector result(data.size() / sizeof(T)); + std::memcpy(result.data(), data.data(), data.size()); + return result; +} +Json tensor(const char* dtype, const Json& sizes, const Json& data) { + return {{"dtype", dtype}, {"sizes", sizes}, {"data", data}}; +} + +TEST(DataInitializer, GeneratesReproducibleIntegerValuedFloatAndHalfInputs) { + const Json fp = + tensor("float32", {512}, {{"rand_uniform_int", {-10, 10}}, {"seed", 23}}); + Json half = fp; + half["dtype"] = "float16"; + const auto data = make_data(fp, 512, 1); + EXPECT_EQ(data, make_data(fp, 512, 99)); + EXPECT_EQ(data, make_data(half, 512, 2)); + bool minimum = false, maximum = false; + for (float value : unpack(data)) { + EXPECT_EQ(std::trunc(value), value); + EXPECT_GE(value, -10); + EXPECT_LE(value, 10); + minimum |= value == -10; + maximum |= value == 10; + } + EXPECT_TRUE(minimum); + EXPECT_TRUE(maximum); +} +TEST(DataInitializer, RejectsFractionalOrOversizedIntegerRandomBounds) { + for (const Json& bounds : + {Json{0, 1.5}, + Json{2, 1}, + Json{0, 4294967296ULL}, + Json{-2147483648LL, 4294967295ULL}}) { + EXPECT_THROW( + validate_data_initializer( + tensor("float32", {8}, {{"rand_uniform_int", bounds}}), 8), + std::invalid_argument); + } +} +TEST(DataInitializer, RepeatsPatternsAndValidatesEveryValue) { + EXPECT_EQ( + unpack( + make_data(tensor("int8", {8}, {{"repeat", {-128, -1, 127}}}), 8)), + (std::vector{-128, -1, 127, -128, -1, 127, -128, -1})); + EXPECT_EQ( + unpack( + make_data(tensor("float16", {5}, {{"repeat", {.5, -1}}}), 5)), + (std::vector{.5f, -1, .5f, -1, .5f})); + for (const Json& pattern : + {Json::array(), + Json("bad"), + Json{1, 128}, + Json{1, 1.5}, + Json{1, nullptr}}) { + EXPECT_THROW( + validate_data_initializer( + tensor("int8", {1}, {{"repeat", pattern}}), 1), + std::invalid_argument); + } +} +TEST(DataInitializer, AbsoluteModuloMatchesRawSignedDraws) { + const Json source = + tensor("int32", {512}, {{"rand_uniform_int", {-100, 100}}, {"seed", 3}}); + const Json derived = tensor( + "int32", + {512}, + {{"abs_mod", {{"bounds", {-100, 100}}, {"mod", 7}, {"seed", 3}}}}); + const auto raw = unpack(make_data(source, 512)); + const auto values = unpack(make_data(derived, 512)); + for (size_t i = 0; i < values.size(); ++i) + EXPECT_EQ(values[i], std::abs(raw[i]) % 7); + EXPECT_EQ( + unpack(make_data( + tensor( + "int32", + {2}, + {{"abs_mod", + {{"bounds", {-2147483648LL, -2147483648LL}}, {"mod", 3}}}}), + 2)), + (std::vector{2, 2})); + EXPECT_EQ( + unpack(make_data( + tensor( + "uint8", + {2}, + {{"abs_mod", {{"bounds", {-1000, -1000}}, {"mod", 7}}}}), + 2)), + (std::vector{6, 6})); +} +TEST(DataInitializer, RejectsInvalidAbsoluteModuloDefinitions) { + for (const Json& definition : + {Json{{"bounds", {0, 1}}, {"mod", 0}}, + Json{{"bounds", {0, 1}}, {"mod", 1.5}}, + Json{{"bounds", {0, 1.5}}, {"mod", 2}}, + Json{{"bounds", {0, 1}}, {"mod", 2}, {"extra", 1}}, + Json{{"bounds", {0, 1}}, {"mod", 2}, {"seed", -1}}}) { + EXPECT_THROW( + validate_data_initializer( + tensor("int32", {4}, {{"abs_mod", definition}}), 4), + std::invalid_argument); + } +} +TEST(DataInitializer, GeneratesEntirePaddedRowsBeforeZeroingColumns) { + const Json initializer = {{"rand_uniform_int", {-128, 127}}, {"seed", 11}}; + const auto raw = + unpack(make_data(tensor("int8", {3, 8}, initializer), 24)); + const Json padded = tensor( + "int8", + {3, 8}, + {{"pad_columns", {{"source", initializer}, {"valid_columns", 5}}}}); + const auto values = unpack(make_data(padded, 24)); + for (size_t i = 0; i < 24; ++i) + EXPECT_EQ(values[i], i % 8 < 5 ? raw[i] : 0); + const auto repeat = unpack(make_data( + tensor( + "float32", + {2, 4}, + {{"pad_columns", + {{"source", {{"repeat", {1, 2, 3}}}}, {"valid_columns", 2}}}}), + 8)); + EXPECT_EQ(repeat, (std::vector{1, 2, 0, 0, 2, 3, 0, 0})); +} +TEST(DataInitializer, RejectsInvalidPaddingShapesAndSources) { + for (const Json& definition : + {Json{{"source", {{"repeat", {1}}}}, {"valid_columns", -1}}, + Json{{"source", {{"repeat", {1}}}}, {"valid_columns", 5}}, + Json{{"source", {{"repeat", {1}}}}, {"valid_columns", 1.5}}, + Json{{"source", {{"row_sum", "other"}}}, {"valid_columns", 1}}, + Json{ + {"source", {{"repeat", {1}}}}, + {"valid_columns", 1}, + {"extra", 1}}}) { + EXPECT_THROW( + validate_data_initializer( + tensor("float32", {2, 4}, {{"pad_columns", definition}}), 8), + std::invalid_argument); + } + EXPECT_THROW( + make_data( + tensor( + "float32", + {2, 4}, + {{"pad_columns", + {{"source", {{"repeat", {1}}}}, {"valid_columns", 1}}}}), + 7), + std::invalid_argument); +} +TEST(DataInitializer, PadsRowSumsAndReducesAllLeadingColumnDimensions) { + const Json specs = { + {"weight", + tensor("int8", {2, 2, 3}, {1, 2, 3, 4, 5, 6, -1, -2, -3, -4, -5, -6})}, + {"rows", tensor("int32", {4}, {{"row_sum", "weight"}})}, + {"columns", tensor("int32", {4}, {{"column_sum", "weight"}})}}; + const auto values = make_tensor_data(specs); + EXPECT_EQ( + unpack(values.at("rows")), + (std::vector{21, -21, 0, 0})); + EXPECT_EQ( + unpack(values.at("columns")), + (std::vector{0, 0, 0, 0})); + Json nonzero = specs; + nonzero["weight"]["data"] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12}; + EXPECT_EQ( + unpack(make_tensor_data(nonzero).at("columns")), + (std::vector{22, 26, 30, 0})); +} +TEST(DataInitializer, RejectsShortSumsUnknownSourcesCyclesAndOverflow) { + Json specs = { + {"weight", tensor("int32", {2, 2}, {1, 2, 3, 4})}, + {"sum", tensor("int32", {1}, {{"row_sum", "weight"}})}}; + EXPECT_THROW(make_tensor_data(specs), std::invalid_argument); + specs["sum"]["data"] = {{"column_sum", "weight"}}; + EXPECT_THROW(make_tensor_data(specs), std::invalid_argument); + specs["sum"]["data"] = {{"row_sum", "missing"}}; + EXPECT_THROW(make_tensor_data(specs), std::invalid_argument); + specs = { + {"a", tensor("int32", {1}, {{"column_sum", "b"}})}, + {"b", tensor("int32", {1}, {{"row_sum", "a"}})}}; + EXPECT_THROW(make_tensor_data(specs), std::invalid_argument); + specs = { + {"weight", tensor("int32", {1, 2}, {2147483647, 1})}, + {"sum", tensor("int32", {1}, {{"row_sum", "weight"}})}}; + EXPECT_THROW(make_tensor_data(specs), std::invalid_argument); + specs["weight"]["dtype"] = "float32"; + EXPECT_THROW(make_tensor_data(specs), std::invalid_argument); +} +TEST(DataInitializer, PackedQ4GroupSumsUseGroupMajorOrderAndPadOutputChannels) { + const Json packed = {0x98, 0x76, 0xAF, 0x80}; + const Json base = { + {"weight", tensor("uint8", {2, 2}, packed)}, + {"sum", + tensor( + "int32", + {2, 4}, + {{"q4_group_sum", {{"tensor", "weight"}, {"group_size", 2}}}})}}; + const auto values = unpack(make_tensor_data(base).at("sum")); + EXPECT_EQ(values, (std::vector{1, 9, 0, 0, -3, -8, 0, 0})); + Json signed_source = base; + signed_source["weight"]["dtype"] = "int8"; + signed_source["weight"]["data"] = {-104, 118, -81, -128}; + EXPECT_EQ(unpack(make_tensor_data(signed_source).at("sum")), values); + for (const char* dtype : {"float32", "float16"}) { + Json fp = base; + fp["sum"]["dtype"] = dtype; + EXPECT_EQ( + unpack(make_tensor_data(fp).at("sum")), + (std::vector{1, 9, 0, 0, -3, -8, 0, 0})); + } +} +TEST(DataInitializer, RejectsInvalidQ4GroupShapesSourcesAndCycles) { + Json specs = { + {"weight", tensor("uint8", {2, 2}, {0, 0, 0, 0})}, + {"sum", + tensor( + "int32", + {2, 2}, + {{"q4_group_sum", {{"tensor", "weight"}, {"group_size", 2}}}})}}; + for (int size : {0, 3, 8}) { + Json invalid = specs; + invalid["sum"]["data"]["q4_group_sum"]["group_size"] = size; + EXPECT_THROW(make_tensor_data(invalid), std::invalid_argument); + } + Json invalid = specs; + invalid["sum"]["sizes"] = {1, 2}; + EXPECT_THROW(make_tensor_data(invalid), std::invalid_argument); + invalid = specs; + invalid["sum"]["sizes"] = {2, 1}; + EXPECT_THROW(make_tensor_data(invalid), std::invalid_argument); + invalid = specs; + invalid["weight"]["dtype"] = "int32"; + EXPECT_THROW(make_tensor_data(invalid), std::invalid_argument); + invalid = specs; + invalid["sum"]["data"]["q4_group_sum"]["tensor"] = "sum"; + EXPECT_THROW(make_tensor_data(invalid), std::invalid_argument); +} +} // namespace +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/test/gpu_clock_test.cpp b/backends/vulkan/test/ops/benchmark_v2/framework/test/gpu_clock_test.cpp new file mode 100644 index 00000000000..c4bf9877877 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/test/gpu_clock_test.cpp @@ -0,0 +1,112 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include + +#include +#include +#include + +namespace vkcompute::shader_profiling { +namespace { + +class GpuClockTest : public ::testing::Test { + protected: + std::filesystem::path root_; + + void SetUp() override { + do { + root_ = std::filesystem::temp_directory_path() / + ("benchmark_gpu_clock_" + std::to_string(std::random_device{}())); + } while (!std::filesystem::create_directory(root_)); + } + + void TearDown() override { + std::filesystem::remove_all(root_); + } + + void write(const std::string& path, const std::string& value) { + const auto file = root_ / path; + std::filesystem::create_directories(file.parent_path()); + std::ofstream output(file); + ASSERT_TRUE(output.good()); + output << value; + } +}; + +TEST_F(GpuClockTest, ReadsChangingAdrenoFrequencyInHz) { + const std::string path = "class/devfreq/3d00000.qcom,kgsl-3d0/cur_freq"; + write(path, "903000000\n"); + GpuClock clock(root_.string()); + EXPECT_EQ(clock.read_mhz(), 903.0); + write(path, "366000000\n"); + EXPECT_EQ(clock.read_mhz(), 366.0); +} + +TEST_F(GpuClockTest, ReadsClassicKgslFrequencyInHz) { + write("class/kgsl/kgsl-3d0/gpuclk", "578000000\n"); + EXPECT_EQ(GpuClock(root_.string()).read_mhz(), 578.0); +} + +TEST_F(GpuClockTest, ReadsPixelMaliFrequencyInKHz) { + write("devices/platform/1f000000.mali/cur_freq", "150000\n"); + EXPECT_EQ(GpuClock(root_.string()).read_mhz(), 150.0); +} + +TEST_F(GpuClockTest, ReadsLegacyMaliClockInMHz) { + write("devices/platform/18500000.mali/clock", "702\n"); + EXPECT_EQ(GpuClock(root_.string()).read_mhz(), 702.0); +} + +TEST_F(GpuClockTest, ReadsExynosSgpuFrequencyInKHz) { + write("class/devfreq/22200000.sgpu/cur_freq", "1000000\n"); + EXPECT_EQ(GpuClock(root_.string()).read_mhz(), 1000.0); +} + +TEST_F(GpuClockTest, KernelFallbackUsesDriverSpecificUnits) { + write("class/devfreq/22200000.sgpu/name", "sgpu\n"); + write("kernel/gpu/gpu_clock", "800000\n"); + EXPECT_EQ(GpuClock(root_.string()).read_mhz(), 800.0); + std::filesystem::remove_all(root_ / "class/devfreq/22200000.sgpu"); + write("class/devfreq/3d00000.qcom,kgsl-3d0/name", "kgsl\n"); + write("kernel/gpu/gpu_clock", "903\n"); + EXPECT_EQ(GpuClock(root_.string()).read_mhz(), 903.0); +} + +TEST_F(GpuClockTest, IgnoresBusAndNonGpuDevfreqDevices) { + write("class/devfreq/kgsl-busmon/cur_freq", "999000000\n"); + write("class/devfreq/gpu-bw/cur_freq", "999000000\n"); + write("class/devfreq/17000010.devfreq_mif/cur_freq", "999000000\n"); + write("class/devfreq/1c000000.gpu/cur_freq", "450000000\n"); + EXPECT_EQ(GpuClock(root_.string()).read_mhz(), 450.0); +} + +TEST_F(GpuClockTest, MissingAndInvalidSourcesAreUnavailable) { + EXPECT_EQ(GpuClock(root_.string()).read_mhz(), std::nullopt); + write("class/kgsl/kgsl-3d0/gpuclk", "invalid"); + EXPECT_EQ(GpuClock(root_.string()).read_mhz(), std::nullopt); + write("class/devfreq/3d00000.qcom,kgsl-3d0/cur_freq", "903000000\n"); + EXPECT_EQ(GpuClock(root_.string()).read_mhz(), 903.0); +} + +TEST_F(GpuClockTest, FailedSamplesAreUnavailableAndZeroIsValid) { + const std::string path = "class/kgsl/kgsl-3d0/gpuclk"; + write(path, "903000000\n"); + GpuClock clock(root_.string()); + for (const char* value : {"", "bad", "-1", "nan", "inf", "100 200"}) { + write(path, value); + EXPECT_EQ(clock.read_mhz(), std::nullopt); + } + write(path, "0\n"); + EXPECT_EQ(clock.read_mhz(), 0.0); + std::filesystem::remove(root_ / path); + EXPECT_EQ(clock.read_mhz(), std::nullopt); +} + +} // namespace +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/test/operator_route_test.cpp b/backends/vulkan/test/ops/benchmark_v2/framework/test/operator_route_test.cpp new file mode 100644 index 00000000000..29dbb35d8e5 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/test/operator_route_test.cpp @@ -0,0 +1,426 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include +#include +#include + +#include + +#include + +#include + +namespace vkcompute::shader_profiling { +using BenchmarkOperatorRoute = VulkanTest; + +namespace { + +struct FpConvCase { + std::vector weights; + std::vector bias; + std::vector half_weights; + std::vector half_bias; + ComputeGraph graph{GraphConfig{}}; + std::vector args; + + explicit FpConvCase( + vkapi::ScalarType dtype = vkapi::kFloat, + utils::StorageType storage = utils::kTexture3D, + utils::GPUMemoryLayout layout = utils::kChannelsPacked, + int64_t batch = 1, + int64_t channels = 4, + int64_t groups = 1) + : weights(8 * (channels / groups) * 9, 0.25f), + bias(8, 0.1f), + half_weights(weights.size(), executorch::runtime::etensor::Half(0.25f)), + half_bias(bias.size(), executorch::runtime::etensor::Half(0.1f)) { + const void* weight_data = dtype == vkapi::kHalf + ? static_cast(half_weights.data()) + : weights.data(); + const void* bias_data = dtype == vkapi::kHalf + ? static_cast(half_bias.data()) + : bias.data(); + args = { + graph.add_tensor({batch, channels, 7, 7}, dtype, storage, layout), + graph.add_tensorref({8, channels / groups, 3, 3}, dtype, weight_data), + graph.add_tensorref({8}, dtype, bias_data), + graph.add_scalar_list(std::vector{1, 1}), + graph.add_scalar_list(std::vector{1, 1}), + graph.add_scalar_list(std::vector{1, 1}), + graph.add_scalar(false), + graph.add_scalar_list(std::vector{0, 0}), + graph.add_scalar(groups), + graph.add_tensor({batch, 8, 7, 7}, dtype, storage, layout)}; + } +}; + +struct Q8ConvCase { + std::vector weights; + std::vector sums; + std::vector scales; + std::vector bias; + ComputeGraph graph{GraphConfig{}}; + std::vector args; + + explicit Q8ConvCase( + int64_t batch = 1, + int64_t channels = 4, + int64_t groups = 1) + : weights(8 * (channels / groups) * 9, 1), + sums(8, int32_t((channels / groups) * 9)), + scales(8, 0.1f), + bias(8, 0.0f) { + args = { + graph.add_tensor( + {batch, channels, 7, 7}, + vkapi::kInt8x4, + utils::kBuffer, + utils::kPackedInt8_4W4C), + graph.add_scalar(0.25), + graph.add_scalar(int64_t(0)), + graph.add_tensorref( + {8, (channels / groups) * 9}, vkapi::kChar, weights.data()), + graph.add_tensorref({8}, vkapi::kInt, sums.data()), + graph.add_tensorref({8}, vkapi::kFloat, scales.data()), + graph.add_scalar(0.25), + graph.add_scalar(int64_t(0)), + graph.add_tensorref({8}, vkapi::kFloat, bias.data()), + graph.add_scalar_list(std::vector{3, 3}), + graph.add_scalar_list(std::vector{1, 1}), + graph.add_scalar_list(std::vector{1, 1}), + graph.add_scalar_list(std::vector{1, 1}), + graph.add_scalar(groups), + graph.add_string("none"), + graph.add_tensor( + {batch, 8, 7, 7}, + vkapi::kInt8x4, + utils::kBuffer, + utils::kPackedInt8_4W4C)}; + } +}; + +TEST_F( + BenchmarkOperatorRoute, + DirectConvolutionBuildsProductionShaderForBothDtypes) { + for (const auto dtype : {vkapi::kFloat, vkapi::kHalf}) { + FpConvCase test(dtype); + const Json selected = + build_conv2d_direct(test.graph, test.args, Json::object()); + EXPECT_EQ(selected.at("name"), "direct"); + EXPECT_TRUE(selected.at("options").empty()); + ASSERT_EQ(test.graph.execute_nodes().size(), 1); + EXPECT_EQ(test.graph.execute_nodes()[0]->name().find("conv2d_"), 0); + EXPECT_EQ( + inspect_conv2d_route(test.graph, test.args, 0, 1).at("name"), "direct"); + } +} + +TEST_F( + BenchmarkOperatorRoute, + Im2colConvolutionHonorsAllIntermediateStorageOverrides) { + for (const char* storage : {"buffer", "texture2d", "texture3d"}) { + FpConvCase test; + const Json selected = build_conv2d_im2col( + test.graph, test.args, {{"intermediate_storage", storage}}); + EXPECT_EQ(selected.at("name"), "im2col"); + EXPECT_EQ(selected.at("options").at("intermediate_storage"), storage); + ASSERT_EQ(test.graph.execute_nodes().size(), 2); + EXPECT_EQ( + test.graph.execute_nodes()[0]->name().find( + std::string("conv2d_im2col_") + storage + "_"), + 0); + EXPECT_EQ( + test.graph.execute_nodes()[1]->name().find( + std::string("conv2d_gemm_") + storage + "_"), + 0); + EXPECT_EQ(inspect_conv2d_route(test.graph, test.args, 0, 2), selected); + } +} + +TEST_F( + BenchmarkOperatorRoute, + ForcedIm2colAllowsDilatedCasesExcludedByAutoPolicy) { + FpConvCase test; + test.args[4] = test.graph.add_scalar_list(std::vector{2, 2}); + test.args[5] = test.graph.add_scalar_list(std::vector{2, 2}); + EXPECT_EQ( + build_conv2d_im2col( + test.graph, test.args, {{"intermediate_storage", "buffer"}}) + .at("name"), + "im2col"); + EXPECT_EQ(test.graph.execute_nodes().size(), 2); +} + +TEST_F( + BenchmarkOperatorRoute, + UnsupportedConvolutionRoutesRejectBeforeAddingNodes) { + FpConvCase batched( + vkapi::kFloat, utils::kTexture3D, utils::kChannelsPacked, 2); + EXPECT_THROW( + build_conv2d_direct(batched.graph, batched.args, Json::object()), + UnsupportedRoute); + EXPECT_THROW( + build_conv2d_im2col(batched.graph, batched.args, Json::object()), + UnsupportedRoute); + EXPECT_TRUE(batched.graph.execute_nodes().empty()); + EXPECT_TRUE(batched.graph.prepack_nodes().empty()); + FpConvCase buffer(vkapi::kFloat, utils::kBuffer); + EXPECT_THROW( + build_conv2d_direct(buffer.graph, buffer.args, Json::object()), + UnsupportedRoute); + EXPECT_THROW( + build_conv2d_im2col(buffer.graph, buffer.args, Json::object()), + UnsupportedRoute); + FpConvCase width(vkapi::kFloat, utils::kTexture3D, utils::kWidthPacked); + EXPECT_THROW( + build_conv2d_direct(width.graph, width.args, Json::object()), + UnsupportedRoute); + FpConvCase grouped( + vkapi::kFloat, utils::kTexture3D, utils::kChannelsPacked, 1, 8, 2); + EXPECT_THROW( + build_conv2d_im2col(grouped.graph, grouped.args, Json::object()), + UnsupportedRoute); + FpConvCase transposed; + transposed.args[6] = transposed.graph.add_scalar(true); + EXPECT_THROW( + build_conv2d_im2col(transposed.graph, transposed.args, Json::object()), + UnsupportedRoute); +} + +TEST_F(BenchmarkOperatorRoute, ForcedDirectSupportsAlignedGroupedConvolution) { + FpConvCase test( + vkapi::kFloat, utils::kTexture3D, utils::kChannelsPacked, 1, 8, 2); + EXPECT_EQ( + build_conv2d_direct(test.graph, test.args, Json::object()).at("name"), + "direct"); + EXPECT_EQ(test.graph.execute_nodes().size(), 1); +} + +TEST_F(BenchmarkOperatorRoute, UnknownOptionsAndGpuWeightsAreRejected) { + FpConvCase test; + EXPECT_THROW( + build_conv2d_direct( + test.graph, test.args, {{"intermediate_storage", "buffer"}}), + std::invalid_argument); + EXPECT_THROW( + build_conv2d_im2col(test.graph, test.args, {{"unknown", true}}), + std::invalid_argument); + EXPECT_THROW( + build_conv2d_im2col( + test.graph, test.args, {{"intermediate_storage", "unknown"}}), + std::invalid_argument); + test.args[1] = test.graph.add_tensor( + {8, 4, 3, 3}, vkapi::kFloat, utils::kTexture3D, utils::kChannelsPacked); + EXPECT_THROW( + build_conv2d_direct(test.graph, test.args, Json::object()), + UnsupportedRoute); + EXPECT_THROW( + build_conv2d_im2col(test.graph, test.args, Json::object()), + UnsupportedRoute); + EXPECT_TRUE(test.graph.execute_nodes().empty()); +} + +TEST_F( + BenchmarkOperatorRoute, + Q8RoutesBuildNativeGeneralAndIm2colWithoutPerfPolicy) { + for (int64_t batch : {int64_t(1), int64_t(2)}) { + Q8ConvCase direct(batch); + EXPECT_EQ( + build_q8ta_conv2d_direct(direct.graph, direct.args, Json::object()) + .at("name"), + "direct"); + ASSERT_EQ(direct.graph.execute_nodes().size(), 1); + EXPECT_EQ(direct.graph.execute_nodes()[0]->name().find("q8ta_conv2d"), 0); + Q8ConvCase im2col(batch); + EXPECT_EQ( + build_q8ta_conv2d_im2col(im2col.graph, im2col.args, Json::object()) + .at("name"), + "im2col"); + ASSERT_EQ(im2col.graph.execute_nodes().size(), 2); + EXPECT_EQ(im2col.graph.execute_nodes()[0]->name(), "q8ta_im2col"); + EXPECT_EQ( + inspect_q8ta_conv2d_route(im2col.graph, im2col.args, 0, 2).at("name"), + "im2col"); + } +} + +TEST_F(BenchmarkOperatorRoute, Q8Im2colRejectsPackingAndUnsupportedOptions) { + Q8ConvCase test; + test.args[0] = test.graph.add_tensor( + {1, 4, 7, 7}, vkapi::kInt8x4, utils::kBuffer, utils::kPackedInt8_4W); + EXPECT_THROW( + build_q8ta_conv2d_im2col(test.graph, test.args, Json::object()), + UnsupportedRoute); + EXPECT_THROW( + build_q8ta_conv2d_direct(test.graph, test.args, {{"unknown", true}}), + std::invalid_argument); + EXPECT_TRUE(test.graph.execute_nodes().empty()); +} + +TEST_F( + BenchmarkOperatorRoute, + RegistryPreservesArgumentsAndReportsRequestedAndResolvedRoutes) { + FpConvCase explicit_case; + const auto original_args = explicit_case.args; + const Json requested = { + {"name", "im2col"}, {"options", {{"intermediate_storage", "texture2d"}}}}; + const Json result = build_operator_route( + {{"operator", "aten.convolution.default"}, {"route", requested}}, + explicit_case.graph, + explicit_case.args); + EXPECT_EQ(result.at("requested"), requested); + EXPECT_EQ(result.at("resolved").at("name"), "im2col"); + EXPECT_EQ( + result.at("resolved").at("options").at("intermediate_storage"), + "texture2d"); + EXPECT_EQ(explicit_case.args, original_args); + EXPECT_EQ( + explicit_case.graph.sizes_of(original_args[0]), + (std::vector{1, 4, 7, 7})); + FpConvCase automatic; + const Json auto_result = build_operator_route( + {{"operator", "aten.convolution.default"}}, + automatic.graph, + automatic.args); + EXPECT_EQ(auto_result.at("requested").at("name"), "auto"); + EXPECT_TRUE(auto_result.at("requested").at("options").empty()); + EXPECT_EQ( + auto_result.at("resolved"), + inspect_conv2d_route( + automatic.graph, + automatic.args, + 0, + automatic.graph.execute_nodes().size())); + EXPECT_NE(auto_result.at("resolved").at("name"), "unknown"); +} + +TEST_F( + BenchmarkOperatorRoute, + RegistryRejectsUnknownNamesAndAutoOptionsWithoutDispatches) { + FpConvCase test; + EXPECT_THROW( + build_operator_route( + {{"operator", "aten.convolution.default"}, + {"route", + {{"name", "not_registered"}, {"options", Json::object()}}}}, + test.graph, + test.args), + std::invalid_argument); + EXPECT_THROW( + build_operator_route( + {{"operator", "aten.convolution.default"}, + {"route", + {{"name", "auto"}, + {"options", {{"intermediate_storage", "buffer"}}}}}}, + test.graph, + test.args), + std::invalid_argument); + EXPECT_TRUE(test.graph.execute_nodes().empty()); + EXPECT_TRUE(test.graph.prepack_nodes().empty()); +} + +TEST_F(BenchmarkOperatorRoute, GroupedRoutesRejectOutputBlocksCrossingGroups) { + FpConvCase fp( + vkapi::kFloat, utils::kTexture3D, utils::kChannelsPacked, 1, 16, 4); + EXPECT_THROW( + build_conv2d_direct(fp.graph, fp.args, Json::object()), UnsupportedRoute); + EXPECT_TRUE(fp.graph.prepack_nodes().empty()); + Q8ConvCase q8(1, 16, 4); + EXPECT_THROW( + build_q8ta_conv2d_direct(q8.graph, q8.args, Json::object()), + UnsupportedRoute); + EXPECT_THROW( + build_q8ta_conv2d_im2col(q8.graph, q8.args, Json::object()), + UnsupportedRoute); + EXPECT_TRUE(q8.graph.execute_nodes().empty()); + EXPECT_TRUE(q8.graph.prepack_nodes().empty()); +} + +TEST_F( + BenchmarkOperatorRoute, + ForcedQ8Im2colDoesNotFallBackWhenScratchPlanIsInfeasible) { + std::vector large_weights(8 * 65536, 1); + Q8ConvCase test; + test.args[0] = test.graph.add_tensor( + {1, 65536, 1, 1024}, + vkapi::kInt8x4, + utils::kBuffer, + utils::kPackedInt8_4W4C); + test.args[3] = + test.graph.add_tensorref({8, 65536}, vkapi::kChar, large_weights.data()); + test.args[9] = test.graph.add_scalar_list(std::vector{1, 1}); + test.args[11] = test.graph.add_scalar_list(std::vector{0, 0}); + test.args[15] = test.graph.add_tensor( + {1, 8, 1, 1024}, vkapi::kInt8x4, utils::kBuffer, utils::kPackedInt8_4W4C); + EXPECT_THROW( + build_q8ta_conv2d_im2col(test.graph, test.args, Json::object()), + UnsupportedRoute); + EXPECT_TRUE(test.graph.execute_nodes().empty()); + EXPECT_TRUE(test.graph.prepack_nodes().empty()); +} + +TEST_F( + BenchmarkOperatorRoute, + FloatingConstantDtypesMustMatchTheirPrepackShader) { + FpConvCase mismatched_bias; + mismatched_bias.args[2] = mismatched_bias.graph.add_tensorref( + {8}, vkapi::kHalf, mismatched_bias.half_bias.data()); + EXPECT_THROW( + build_conv2d_direct( + mismatched_bias.graph, mismatched_bias.args, Json::object()), + UnsupportedRoute); + EXPECT_THROW( + build_conv2d_im2col( + mismatched_bias.graph, mismatched_bias.args, Json::object()), + UnsupportedRoute); + EXPECT_TRUE(mismatched_bias.graph.prepack_nodes().empty()); + FpConvCase half_io(vkapi::kHalf); + if (half_io.graph.dtype_of(half_io.args[0]) == vkapi::kHalf) { + half_io.args[1] = half_io.graph.add_tensorref( + {8, 4, 3, 3}, vkapi::kFloat, half_io.weights.data()); + half_io.args[2] = + half_io.graph.add_tensorref({8}, vkapi::kFloat, half_io.bias.data()); + EXPECT_THROW( + build_conv2d_direct(half_io.graph, half_io.args, Json::object()), + UnsupportedRoute); + } +} + +} // namespace +TEST_F( + BenchmarkOperatorRoute, + LegacyQdqRejectsUnsupportedFloatingBufferLayout) { + ComputeGraph graph{GraphConfig{}}; + const auto fp = graph.add_tensor( + {1, 3, 15, 15}, vkapi::kFloat, utils::kBuffer, utils::kChannelsPacked); + const auto quant = graph.add_tensor( + {1, 3, 15, 15}, vkapi::kInt8x4, utils::kBuffer, utils::kPackedInt8_4W4C); + const auto scale = graph.add_scalar(0.1); + const auto zero = graph.add_scalar(int64_t(0)); + const auto qmin = graph.add_scalar(int64_t(-128)); + const auto qmax = graph.add_scalar(int64_t(127)); + const auto dtype = graph.add_scalar(int64_t(2)); + const auto out_dtype = graph.add_scalar(int64_t(10)); + const auto quantize = route_registry().operator_route( + "quantized_decomposed.quantize_per_tensor.default", "legacy_4w4c"); + const auto dequantize = route_registry().operator_route( + "quantized_decomposed.dequantize_per_tensor.default", "legacy_4w4c"); + EXPECT_THROW( + quantize( + graph, {fp, scale, zero, qmin, qmax, dtype, quant}, Json::object()), + UnsupportedRoute); + EXPECT_THROW( + dequantize( + graph, + {quant, scale, zero, qmin, qmax, dtype, out_dtype, fp}, + Json::object()), + UnsupportedRoute); + EXPECT_TRUE(graph.execute_nodes().empty()); +} + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/test/reference_registry_test.cpp b/backends/vulkan/test/ops/benchmark_v2/framework/test/reference_registry_test.cpp new file mode 100644 index 00000000000..637caaf4343 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/test/reference_registry_test.cpp @@ -0,0 +1,251 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include + +#include +#include +#include + +namespace vkcompute::shader_profiling { +namespace { + +reference::Tensor +identity_reference(const Json&, const reference::TensorMap& tensors, size_t) { + return tensors.at("x"); +} + +reference::Tensor offset_reference( + const Json&, + const reference::TensorMap& tensors, + size_t threads) { + reference::Tensor result = tensors.at("x"); + for (double& value : result.values) { + value += threads; + } + return result; +} + +ReferenceOutputs +multiple_reference(const Json&, const reference::TensorMap&, size_t) { + return {{"scale", {0.12345}}, {"zero_point", {-3}}}; +} + +REGISTER_REFERENCE_FUNCTIONS { + REGISTER_REFERENCE_FN(test_identity, identity_reference); + REGISTER_REFERENCE_OUTPUTS_FN(test_multiple, multiple_reference); + REGISTER_REFERENCE_FN(test_offset, offset_reference); + REGISTER_OPERATOR_REFERENCE(aten.test_reference.default, test_identity); + REGISTER_SHADER_REFERENCE(test_identity_shader, test_identity); +} + +TEST(BenchmarkReferenceRegistry, RegisteredNamesSelectCallables) { + ReferenceRegistry registry; + registry.register_reference("identity", identity_reference); + registry.register_reference("offset", offset_reference); + EXPECT_EQ(registry.get_reference("identity"), identity_reference); + EXPECT_EQ(registry.get_reference("offset"), offset_reference); +} + +TEST(BenchmarkReferenceRegistry, MultipleNamesCanShareAReference) { + ReferenceRegistry registry; + registry.register_reference("identity", identity_reference); + registry.register_operator_reference("aten.first.default", "identity"); + registry.register_operator_reference("aten.second.default", "identity"); + registry.register_shader_reference("shader_a", "identity"); + registry.register_shader_reference("shader_b", "identity"); + EXPECT_EQ(registry.operator_reference("aten.first.default"), "identity"); + EXPECT_EQ(registry.operator_reference("aten.second.default"), "identity"); + EXPECT_EQ(registry.shader_reference("shader_a"), "identity"); + EXPECT_EQ(registry.shader_reference("shader_b"), "identity"); +} + +TEST(BenchmarkReferenceRegistry, AssociationsCanPrecedeFunctionRegistration) { + ReferenceRegistry registry; + registry.register_operator_reference("aten.later.default", "later"); + registry.register_shader_reference("later_shader", "later"); + EXPECT_THROW( + registry.operator_reference("aten.later.default"), std::invalid_argument); + EXPECT_THROW( + registry.shader_reference("later_shader"), std::invalid_argument); + registry.register_reference("later", identity_reference); + EXPECT_EQ(registry.operator_reference("aten.later.default"), "later"); + EXPECT_EQ(registry.shader_reference("later_shader"), "later"); +} + +TEST(BenchmarkReferenceRegistry, OperatorAndShaderNamespacesAreSeparate) { + ReferenceRegistry registry; + registry.register_reference("identity", identity_reference); + registry.register_reference("offset", offset_reference); + registry.register_operator_reference("same_name", "identity"); + registry.register_shader_reference("same_name", "offset"); + EXPECT_EQ(registry.operator_reference("same_name"), "identity"); + EXPECT_EQ(registry.shader_reference("same_name"), "offset"); +} + +TEST(BenchmarkReferenceRegistry, DuplicateFunctionNamesKeepOriginalFunction) { + ReferenceRegistry registry; + registry.register_reference("identity", identity_reference); + EXPECT_THROW( + registry.register_reference("identity", offset_reference), + std::invalid_argument); + EXPECT_EQ(registry.get_reference("identity"), identity_reference); +} + +TEST(BenchmarkReferenceRegistry, DuplicateAssociationsKeepOriginalMapping) { + ReferenceRegistry registry; + registry.register_reference("identity", identity_reference); + registry.register_reference("offset", offset_reference); + registry.register_operator_reference("aten.same.default", "identity"); + registry.register_shader_reference("same_shader", "identity"); + EXPECT_THROW( + registry.register_operator_reference("aten.same.default", "offset"), + std::invalid_argument); + EXPECT_THROW( + registry.register_shader_reference("same_shader", "offset"), + std::invalid_argument); + EXPECT_EQ(registry.operator_reference("aten.same.default"), "identity"); + EXPECT_EQ(registry.shader_reference("same_shader"), "identity"); +} + +TEST(BenchmarkReferenceRegistry, MissingFunctionsAndMappingsFailLookup) { + ReferenceRegistry registry; + EXPECT_THROW(registry.get_reference("unknown"), std::invalid_argument); + EXPECT_THROW(registry.operator_reference("unknown"), std::invalid_argument); + EXPECT_THROW(registry.shader_reference("unknown"), std::invalid_argument); +} + +TEST(BenchmarkReferenceRegistry, RejectsEmptyNamesAndNullFunctions) { + ReferenceRegistry registry; + EXPECT_THROW( + registry.register_reference("", identity_reference), + std::invalid_argument); + EXPECT_THROW( + registry.register_reference("null", nullptr), std::invalid_argument); + EXPECT_THROW( + registry.register_operator_reference("", "identity"), + std::invalid_argument); + EXPECT_THROW( + registry.register_operator_reference("op", ""), std::invalid_argument); + EXPECT_THROW( + registry.register_shader_reference("", "identity"), + std::invalid_argument); + EXPECT_THROW( + registry.register_shader_reference("shader", ""), std::invalid_argument); +} + +Json registered_case(bool shader) { + Json spec = { + {"tensors", {{"x", {{"sizes", {2}}}}, {"out", {{"sizes", {2}}}}}}}; + if (shader) { + spec["shader"] = "test_identity_shader"; + spec["bindings"] = {{{"tensor", "out"}}, {{"tensor", "x"}}}; + } else { + spec["operator"] = "aten.test_reference.default"; + spec["args"] = {{{"tensor", "x"}}, {{"tensor", "out"}}}; + } + return spec; +} + +TEST( + BenchmarkReferenceRegistry, + StaticAssociationsDispatchRegisteredFunctions) { + const std::vector values = {1.5f, -2.0f}; + std::vector bytes(values.size() * sizeof(float)); + std::memcpy(bytes.data(), values.data(), bytes.size()); + for (bool shader : {false, true}) { + const Json spec = registered_case(shader); + EXPECT_EQ(reference_name(spec), "test_identity"); + EXPECT_EQ( + reference_outputs(spec, {}, {{"x", bytes}}).at("out"), + (std::vector{1.5, -2.0})); + } +} + +TEST( + BenchmarkReferenceRegistry, + JsonOverrideSelectsCallableAndForwardsThreads) { + Json spec = registered_case(false); + spec["reference"] = "test_offset"; + const std::vector values = {1, 2}; + std::vector bytes(values.size() * sizeof(float)); + std::memcpy(bytes.data(), values.data(), bytes.size()); + EXPECT_EQ(reference_name(spec), "test_offset"); + EXPECT_EQ( + reference_outputs(spec, {}, {{"x", bytes}}, 3).at("out"), + (std::vector{4, 5})); + spec["reference"] = "not_registered"; + EXPECT_THROW(reference_name(spec), std::invalid_argument); +} + +TEST(BenchmarkReferenceRegistry, BuiltinRegistrationsAreLinkedFromFamilyFiles) { + const auto& registry = reference_registry(); + EXPECT_EQ(registry.operator_reference("aten.linear.default"), "linear"); + EXPECT_EQ(registry.operator_reference("aten.convolution.default"), "conv2d"); + EXPECT_EQ( + registry.operator_reference("et_vk.q8ta_conv2d.default"), "q8ta_conv2d"); + EXPECT_EQ( + registry.operator_reference("et_vk.q4gsw_linear.default"), + "q4gsw_linear"); + EXPECT_EQ(registry.shader_reference("buffer_to_buffer_float"), "copy"); + EXPECT_EQ(registry.shader_reference("buffer_to_buffer_half"), "copy"); + EXPECT_EQ(registry.shader_reference("binary_add_texture3d_float"), "add"); + EXPECT_EQ(registry.shader_reference("binary_add_texture3d_half"), "add"); +} + +} // namespace +TEST(BenchmarkReferenceRegistry, MultiOutputReferencesShareTheNameNamespace) { + ReferenceRegistry registry; + registry.register_reference_outputs("multiple", multiple_reference); + registry.register_operator_reference("multi_op", "multiple"); + EXPECT_TRUE(registry.has_reference("multiple")); + EXPECT_EQ(registry.operator_reference("multi_op"), "multiple"); + EXPECT_EQ(registry.get_reference_outputs("multiple"), multiple_reference); + EXPECT_THROW( + registry.register_reference("multiple", identity_reference), + std::invalid_argument); + EXPECT_THROW( + registry.register_reference_outputs("multiple", multiple_reference), + std::invalid_argument); + registry.register_reference("single", identity_reference); + EXPECT_EQ(registry.get_reference_outputs("single"), nullptr); + EXPECT_THROW( + registry.register_reference_outputs("single", multiple_reference), + std::invalid_argument); + EXPECT_THROW( + registry.register_reference_outputs("", multiple_reference), + std::invalid_argument); + EXPECT_THROW( + registry.register_reference_outputs("null", nullptr), + std::invalid_argument); +} + +TEST( + BenchmarkReferenceRegistry, + MultiOutputReferencesValidateCountsAndRoundPerDtype) { + Json spec{ + {"operator", "test_multi"}, + {"reference", "test_multiple"}, + {"tensors", + {{"scale", {{"sizes", {1}}, {"dtype", "float16"}}}, + {"zero_point", {{"sizes", {1}}, {"dtype", "int8"}}}}}}; + const auto results = reference_outputs(spec, Json::object(), {}); + ASSERT_EQ(results.size(), 2); + EXPECT_EQ( + results.at("scale")[0], + float(executorch::runtime::etensor::Half(0.12345f))); + EXPECT_EQ(results.at("zero_point")[0], -3); + spec["tensors"]["scale"]["sizes"] = {2}; + EXPECT_THROW( + reference_outputs(spec, Json::object(), {}), std::invalid_argument); + spec["tensors"].erase("scale"); + EXPECT_THROW( + reference_outputs(spec, Json::object(), {}), std::invalid_argument); +} + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/test/reference_test.cpp b/backends/vulkan/test/ops/benchmark_v2/framework/test/reference_test.cpp new file mode 100644 index 00000000000..40eb28e1e02 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/test/reference_test.cpp @@ -0,0 +1,362 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include + +#include + +#include +#include +#include + +namespace vkcompute::shader_profiling { +namespace { + +Json tensor(const Json& sizes, const std::string& dtype = "float32") { + return {{"sizes", sizes}, {"dtype", dtype}, {"storage", "buffer"}}; +} + +template +std::vector bytes(const std::vector& values) { + std::vector result(values.size() * sizeof(T)); + std::memcpy(result.data(), values.data(), result.size()); + return result; +} + +Json linear_spec() { + return { + {"operator", "aten.linear.default"}, + {"args", + {{{"tensor", "x"}}, + {{"tensor", "w"}}, + {{"tensor", "b"}}, + {{"tensor", "y"}}}}, + {"tensors", + {{"x", tensor({2, 3})}, + {"w", tensor({2, 3})}, + {"b", tensor({2})}, + {"y", tensor({2, 2})}}}}; +} + +TEST(BenchmarkReference, LinearComputesBatchedProjectionWithBias) { + const auto result = reference_outputs( + linear_spec(), + {}, + {{"x", bytes({1, 2, 3, -1, 0, 2})}, + {"w", bytes({2, -1, 1, 0, 3, -2})}, + {"b", bytes({0.5, -0.5})}}, + 2); + EXPECT_EQ(result.at("y"), (std::vector{3.5, -0.5, 0.5, -4.5})); +} + +TEST(BenchmarkReference, LinearSupportsLeadingDimensionsAndNoBias) { + Json spec = linear_spec(); + spec["tensors"]["x"]["sizes"] = {1, 2, 3}; + spec["tensors"]["y"]["sizes"] = {1, 2, 2}; + spec["args"][2] = nullptr; + const auto result = reference_outputs( + spec, + {}, + {{"x", bytes({1, 2, 3, -1, 0, 2})}, + {"w", bytes({2, -1, 1, 0, 3, -2})}}); + EXPECT_EQ(result.at("y"), (std::vector{3, 0, 0, -4})); +} + +TEST( + BenchmarkReference, + LinearTilesAndTailsMatchScalarOracleAcrossThreadCounts) { + constexpr size_t kM = 7; + constexpr size_t kK = 17; + constexpr size_t kN = 129; + Json spec = linear_spec(); + spec["tensors"]["x"]["sizes"] = {kM, kK}; + spec["tensors"]["w"]["sizes"] = {kN, kK}; + spec["tensors"]["b"]["sizes"] = {kN}; + spec["tensors"]["y"]["sizes"] = {kM, kN}; + std::vector input(kM * kK); + std::vector weight(kN * kK); + std::vector bias(kN); + for (size_t i = 0; i < input.size(); ++i) { + input[i] = int(i % 7) - 3; + } + for (size_t i = 0; i < weight.size(); ++i) { + weight[i] = int(i % 11) - 5; + } + for (size_t i = 0; i < bias.size(); ++i) { + bias[i] = int(i % 3) - 1; + } + std::vector expected(kM * kN); + for (size_t row = 0; row < kM; ++row) { + for (size_t column = 0; column < kN; ++column) { + double value = bias[column]; + for (size_t inner = 0; inner < kK; ++inner) { + value += double(input[row * kK + inner]) * + double(weight[column * kK + inner]); + } + expected[row * kN + column] = value; + } + } + const std::map> data = { + {"x", bytes(input)}, {"w", bytes(weight)}, {"b", bytes(bias)}}; + const auto single_thread = reference_outputs(spec, {}, data, 1); + const auto eight_threads = reference_outputs(spec, {}, data, 8); + EXPECT_EQ(single_thread.at("y"), expected); + EXPECT_EQ(eight_threads, single_thread); +} + +TEST(BenchmarkReference, HalfInputsAreRoundedAndPromotedInputsStayFloat) { + Json spec = linear_spec(); + spec["args"][2] = nullptr; + spec["tensors"]["x"] = tensor({1, 1}, "float16"); + spec["tensors"]["w"] = tensor({1, 1}, "float16"); + spec["tensors"]["w"]["storage"] = "constant"; + spec["tensors"]["y"] = tensor({1, 1}, "float16"); + using Half = executorch::runtime::etensor::Half; + const std::map> data = { + {"x", bytes({1.0004f})}, {"w", bytes({Half(1.0f)})}}; + EXPECT_EQ(reference_outputs(spec, {}, data).at("y")[0], 1.0); + EXPECT_EQ( + reference_outputs(spec, {{"x", "float32"}, {"y", "float32"}}, data) + .at("y")[0], + double(1.0004f)); +} + +TEST(BenchmarkReference, LinearRejectsMismatchedOutputShape) { + Json spec = linear_spec(); + spec["tensors"]["y"]["sizes"] = {4, 1}; + EXPECT_THROW( + reference_outputs( + spec, + {}, + {{"x", bytes({1, 2, 3, -1, 0, 2})}, + {"w", bytes({2, -1, 1, 0, 3, -2})}, + {"b", bytes({0, 0})}}), + std::invalid_argument); +} + +TEST(BenchmarkReference, GroupedConvolutionHonorsStridePaddingAndDilation) { + const Json spec = { + {"operator", "aten.convolution.default"}, + {"args", + {{{"tensor", "x"}}, + {{"tensor", "w"}}, + {{"tensor", "b"}}, + {{"int_list", {1, 2}}}, + {{"int_list", {0, 1}}}, + {{"int_list", {1, 2}}}, + {{"bool", false}}, + {{"int_list", {0, 0}}}, + {{"int", 2}}, + {{"tensor", "y"}}}}, + {"tensors", + {{"x", tensor({1, 2, 1, 5})}, + {"w", tensor({2, 1, 1, 2})}, + {"b", tensor({2})}, + {"y", tensor({1, 2, 1, 3})}}}}; + const auto result = reference_outputs( + spec, + {}, + {{"x", bytes({1, 2, 3, 4, 5, 10, 20, 30, 40, 50})}, + {"w", bytes({1, 2, -1, 1})}, + {"b", bytes({1, -1})}}, + 2); + EXPECT_EQ(result.at("y"), (std::vector{5, 11, 5, 19, 19, -41})); +} + +TEST(BenchmarkReference, Q4LinearUnpacksNibblesAndUsesPerGroupScales) { + const Json spec = { + {"operator", "et_vk.q4gsw_linear.default"}, + {"args", + {{{"tensor", "x"}}, + {{"tensor", "w"}}, + {{"tensor", "scales"}}, + {{"int", 2}}, + {{"tensor", "b"}}, + {{"tensor", "y"}}}}, + {"tensors", + {{"x", tensor({1, 4})}, + {"w", tensor({2, 2}, "uint8")}, + {"scales", tensor({2, 2})}, + {"b", tensor({2})}, + {"y", tensor({1, 2})}}}}; + const auto result = reference_outputs( + spec, + {}, + {{"x", bytes({1, 2, 3, 4})}, + {"w", bytes({0x98, 0x76, 0xAF, 0x80})}, + {"scales", bytes({1, 2, 3, 4})}, + {"b", bytes({0.5, -0.5})}}); + EXPECT_EQ(result.at("y"), (std::vector{-27.5, -74.5})); +} + +TEST(BenchmarkReference, Q8ConvolutionAppliesZeroPointsScalesBiasAndClamping) { + const Json spec = { + {"operator", "et_vk.q8ta_conv2d.default"}, + {"args", + {{{"tensor", "x"}}, + {{"float", 0.5}}, + {{"int", 2}}, + {{"tensor", "w"}}, + {{"tensor", "sums"}}, + {{"tensor", "scales"}}, + {{"float", 0.25}}, + {{"int", -3}}, + {{"tensor", "b"}}, + {{"int_list", {1, 1}}}, + {{"int_list", {1, 1}}}, + {{"int_list", {0, 0}}}, + {{"int_list", {1, 1}}}, + {{"int", 1}}, + {{"string", "none"}}, + {{"tensor", "y"}}}}, + {"tensors", + {{"x", tensor({1, 2, 1, 2}, "int8x4")}, + {"w", tensor({2, 2}, "int8")}, + {"sums", tensor({2}, "int32")}, + {"scales", tensor({2})}, + {"b", tensor({2})}, + {"y", tensor({1, 2, 1, 2}, "int8x4")}}}}; + const auto result = reference_outputs( + spec, + {}, + {{"x", bytes({4, -2, 8, 3})}, + {"w", bytes({1, 2, -2, 1})}, + {"sums", bytes({3, -1})}, + {"scales", bytes({0.25, 20})}, + {"b", bytes({0.5, -1})}}); + EXPECT_EQ(result.at("y"), (std::vector{6, -2, 73, 127})); + Json relu = spec; + relu["args"][14]["string"] = "relu"; + EXPECT_EQ( + reference_outputs( + relu, + {}, + {{"x", bytes({0, 0, 0, 0})}, + {"w", bytes({1, 2, -2, 1})}, + {"sums", bytes({3, -1})}, + {"scales", bytes({0.25, 20})}, + {"b", bytes({0, 0})}}) + .at("y"), + (std::vector{-3, -3, 77, 77})); +} +TEST(BenchmarkReference, RawAdditionBroadcastsAndAppliesAlpha) { + const Json spec = { + {"shader", "binary_add_texture3d_float"}, + {"bindings", + {{{"tensor", "out"}, {"access", "write"}}, + {{"tensor", "a"}}, + {{"tensor", "b"}}}}, + {"push_constants", {{{"dtype", "float32"}, {"data", {2}}}}}, + {"tensors", + {{"out", tensor({2, 3})}, {"a", tensor({2, 1})}, {"b", tensor({3})}}}}; + EXPECT_EQ( + reference_outputs( + spec, + {}, + {{"a", bytes({1, 10})}, {"b", bytes({2, 3, 4})}}) + .at("out"), + (std::vector{5, 7, 9, 14, 16, 18})); +} + +TEST(BenchmarkReference, CopyPreservesLogicalValues) { + const Json spec = { + {"shader", "buffer_to_buffer_float"}, + {"bindings", + {{{"tensor", "out"}, {"access", "write"}}, {{"tensor", "in"}}}}, + {"tensors", {{"out", tensor({4})}, {"in", tensor({4})}}}}; + EXPECT_EQ( + reference_outputs(spec, {}, {{"in", bytes({-1, 0, 2, 4})}}) + .at("out"), + (std::vector{-1, 0, 2, 4})); +} + +TEST(BenchmarkReference, ExplicitReferenceOverridesUnknownShader) { + const Json spec = {{"shader", "custom"}, {"reference", "copy"}}; + EXPECT_EQ(reference_name(spec), "copy"); + EXPECT_THROW(reference_name({{"shader", "custom"}}), std::invalid_argument); +} + +TEST(BenchmarkReference, ComparisonReportsBoundedMismatchSummary) { + Json spec = linear_spec(); + spec["validation_method"] = { + {"function", "is_close"}, {"atol", 0.1}, {"rtol", 0.01}}; + const Report result = compare_outputs( + spec, {{"y", {1, 2, 3, 4}}}, {{"y", {1.05, 2.5, 3, 4.5}}}, {}); + EXPECT_EQ(result.at("status"), "FAIL"); + EXPECT_EQ(result.at("outputs").at("y").at("mismatch_count"), 2); + EXPECT_DOUBLE_EQ(result.at("outputs").at("y").at("max_abs_error"), 0.5); + EXPECT_FALSE(result.at("outputs").at("y").contains("values")); + EXPECT_EQ( + compare_outputs(spec, {{"y", {1, 2, 3, 4}}}, {{"y", {1.05, 2, 3, 4}}}, {}) + .at("status"), + "PASS"); +} + +TEST(BenchmarkReference, SelectedOutputsDoNotHideMissingReferenceResults) { + Json spec = linear_spec(); + const ReferenceOutputs expected{{"y", {1, 2, 3, 4}}, {"b", {1, 2}}}; + const ReferenceOutputs actual{{"y", {1, 2, 3, 4}}, {"b", {10, 20}}}; + EXPECT_EQ(compare_outputs(spec, expected, actual, {}).at("status"), "FAIL"); + spec["validation_method"] = {{"function", "is_close"}, {"outputs", {"y"}}}; + const Report selected = compare_outputs(spec, expected, actual, {}); + EXPECT_EQ(selected.at("status"), "PASS"); + EXPECT_EQ(selected.at("outputs").size(), 1); + EXPECT_EQ( + compare_outputs(spec, expected, {{"b", {1, 2}}}, {}).at("status"), + "FAIL"); + EXPECT_EQ( + compare_outputs(spec, {{"b", {1, 2}}}, actual, {}).at("status"), "FAIL"); +} + +TEST(BenchmarkReference, ComparisonRejectsMissingShortAndUnexpectedOutputs) { + const Json spec = linear_spec(); + const ReferenceOutputs expected = {{"y", {1, 2, 3, 4}}}; + EXPECT_EQ(compare_outputs(spec, expected, {}, {}).at("status"), "FAIL"); + EXPECT_EQ( + compare_outputs(spec, expected, {{"y", {1}}}, {}).at("status"), "FAIL"); + EXPECT_EQ( + compare_outputs(spec, expected, {{"y", {1, 2, 3, 4}}, {"extra", {1}}}, {}) + .at("status"), + "FAIL"); +} + +TEST(BenchmarkReference, ComparisonRejectsNonfiniteValues) { + const Json spec = linear_spec(); + const double nan = std::numeric_limits::quiet_NaN(); + const double inf = std::numeric_limits::infinity(); + EXPECT_EQ( + compare_outputs( + spec, {{"y", {1, 2, 3, 4}}}, {{"y", {1, nan, inf, 4}}}, {}) + .at("status"), + "FAIL"); + EXPECT_EQ( + compare_outputs( + spec, {{"y", {nan, 2, 3, 4}}}, {{"y", {nan, 2, 3, 4}}}, {}) + .at("status"), + "FAIL"); +} + +TEST(BenchmarkReference, ComparisonDefaultsDependOnActualDtype) { + Json spec = linear_spec(); + spec["tensors"]["y"]["dtype"] = "float16"; + const ReferenceOutputs expected = {{"y", {1, 2, 3, 4}}}; + const ReferenceOutputs close = {{"y", {1.005, 2, 3, 4}}}; + EXPECT_EQ(compare_outputs(spec, expected, close, {}).at("status"), "PASS"); + EXPECT_EQ( + compare_outputs(spec, expected, close, {{"y", "float32"}}).at("status"), + "FAIL"); + EXPECT_EQ( + compare_outputs(spec, expected, close, {{"y", "int8x4"}}).at("status"), + "FAIL"); + spec["validation_method"] = {{"function", "is_close"}, {"atol", -1}}; + EXPECT_THROW( + compare_outputs(spec, expected, close, {}), std::invalid_argument); +} + +} // namespace +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/test/report_test.cpp b/backends/vulkan/test/ops/benchmark_v2/framework/test/report_test.cpp new file mode 100644 index 00000000000..7550d0f650f --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/test/report_test.cpp @@ -0,0 +1,980 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include + +#include +#include + +namespace vkcompute::shader_profiling { +namespace { + +TEST(BenchmarkReport, ShaderBreakdownKeepsDispatchOrderAndDuplicateShaders) { + const Report dispatches = Report::array( + {{{"dispatch_index", 0}, + {"shader_name", "copy"}, + {"gwg_size", {{"method", "operator"}, {"value", {8, 1, 1}}}}, + {"lwg_size", {{"method", "operator"}, {"value", {4, 1, 1}}}}}, + {{"dispatch_index", 1}, + {"shader_name", "copy"}, + {"gwg_size", {{"method", "operator"}, {"value", {16, 1, 1}}}}, + {"lwg_size", {{"method", "operator"}, {"value", {8, 1, 1}}}}}}); + const Report breakdown = + shader_breakdown_report(dispatches, {{1, 10}, {2, 20}, {3, 30}}); + ASSERT_EQ(breakdown.size(), 2); + EXPECT_EQ(breakdown.at(0).at("shader_name"), "copy"); + EXPECT_EQ(breakdown.at(1).at("dispatch_index"), 1); + EXPECT_EQ( + breakdown.at(1).at("gwg_size").at("value"), Report::array({16, 1, 1})); + EXPECT_EQ(breakdown.at(0).at("gpu_timestamp_shader_us").at("mean"), 2); + EXPECT_NEAR( + breakdown.at(1).at("gpu_timestamp_shader_us").at("p90").get(), + 28, + 1e-12); + EXPECT_TRUE(shader_breakdown_report(Report::array(), {{}, {}}).empty()); + const Report averaged = shader_breakdown_report( + dispatches, {{1, 10, 3, 30}, {2, 20, 4, 40}, {3, 30, 5, 50}}, 2); + EXPECT_EQ(averaged.at(0).at("gpu_timestamp_shader_us").at("mean"), 3); + EXPECT_EQ(averaged.at(1).at("gpu_timestamp_shader_us").at("p90"), 38); + EXPECT_THROW( + shader_breakdown_report(dispatches, {{1}}), std::invalid_argument); + EXPECT_THROW( + shader_breakdown_report(dispatches, {}, 0), std::invalid_argument); +} + +TEST(BenchmarkReport, TableShowsShaderBreakdownUnderOperator) { + Report report = { + {"operator_name", "aten.linear.default"}, + {"args", Report::array()}, + {"shader_breakdown", + Report::array( + {{{"shader_name", "linear_shader"}, + {"gwg_size", {{"method", "operator"}, {"value", {8, 1, 1}}}}, + {"lwg_size", {{"method", "operator"}, {"value", {4, 1, 1}}}}, + {"gpu_timestamp_shader_us", {{"mean", 1.25}, {"p90", 2.5}}}}})}}; + report.update(measurement_report(1, true, {5, 10}, {})); + const std::string table = format_table(Report::array({report})); + EXPECT_NE(table.find(" linear_shader"), std::string::npos); + EXPECT_LT(table.find("aten.linear.default"), table.find("linear_shader")); + EXPECT_NE(table.find("[8,1,1]"), std::string::npos); + EXPECT_NE(table.find("[4,1,1]"), std::string::npos); + EXPECT_NE(table.find("1.250"), std::string::npos); + EXPECT_NE(table.find("2.500"), std::string::npos); +} + +TEST(BenchmarkReport, TableShowsResolvedSizesInputsAndPerShaderStatistics) { + Report report = { + {"shader_name", "add"}, + {"gwg_size", {{"method", "default_pick_gwg"}, {"value", {8, 2, 1}}}}, + {"lwg_size", {{"method", "explicit"}, {"value", {4, 2, 1}}}}, + {"bindings", + {{"out", + {{"type", "buffer"}, + {"access", "write"}, + {"sizes", {99}}, + {"dtype", "int32"}}}, + {"a", + {{"type", "texture3d"}, + {"access", "read"}, + {"sizes", {1, 1, 1, 8}}, + {"dtype", "float32"}}}, + {"b", + {{"type", "buffer"}, + {"access", "read_write"}, + {"sizes", {8}}, + {"dtype", "float16"}}}, + {"meta", {{"type", "meta_ubo"}}}}}}; + report.update(measurement_report(10, false, {10, 20, 30}, {})); + const std::string table = format_table(Report::array({report})); + EXPECT_EQ( + table, + "Shader / operator GWG LWG Inputs Mean GPU us P90 GPU us\n" + "-------------------------------------------------------------------------------------------------\n" + "add [8,2,1] [4,2,1] t3[1,1,1,8]f32.width, b[8]f16.width 2.000 2.800"); +} + +TEST(BenchmarkReport, TableSupportsOperatorsAndUnavailableTimestamps) { + Report report = { + {"operator_name", "aten.linear.default"}, + {"args", + Report::array( + {{{"tensor", + {{"sizes", {16, 32}}, {"dtype", "int8"}, {"access", "read"}}}}, + nullptr, + {{"int", 32}}, + {{"tensor", + {{"sizes", {4, 32}}, + {"dtype", "float32"}, + {"requested_dtype", "float16"}, + {"access", "read_write"}}}}, + {{"tensor", + {{"sizes", {4, 16}}, + {"dtype", "float32"}, + {"access", "write"}}}}})}}; + report.update(measurement_report(2, true, {}, {4})); + const std::string table = format_table(Report::array({report})); + EXPECT_NE( + table.find("b[16,32]i8.width, null, 32, b[4,32]f32.width"), + std::string::npos); + EXPECT_EQ(table.find("[4,16]"), std::string::npos); + EXPECT_EQ(table.find("f16"), std::string::npos); + EXPECT_NE(table.find("N/A"), std::string::npos); +} + +TEST(BenchmarkReport, TableIncludesShaderUniformsPushConstantsAndSweepIndexes) { + const Json spec = { + {"shader", "add"}, + {"sweep_index", 3}, + {"gwg_size", {8, 1, 1}}, + {"lwg_size", {4, 1, 1}}, + {"tensors", Json::object()}, + {"bindings", + Json::array( + {{{"uniform", + Json::array({{{"dtype", "int32"}, {"data", {4, 5}}}})}}})}, + {"push_constants", Json::array({{{"dtype", "float32"}, {"data", {2}}}})}}; + Report report = shader_report( + spec, {8, 1, 1}, {4, 1, 1}, Json::object(), Report::object()); + report.update(measurement_report(1, false, {2}, {})); + const Report error = error_report(spec, "bad shader"); + EXPECT_EQ(report.at("sweep_index"), 3); + EXPECT_EQ(error.at("sweep_index"), 3); + const std::string table = format_table(Report::array({report, error})); + EXPECT_NE(table.find("add [3]"), std::string::npos); + EXPECT_NE(table.find("[4,5], [2]"), std::string::npos); + EXPECT_NE(table.find("ERROR add [3]: bad shader"), std::string::npos); +} + +TEST(BenchmarkReport, TableIncludesScalarInputsAndAllStorageTypes) { + Report report = { + {"operator_name", "aten.convolution.default"}, + {"args", + Report::array( + {{{"tensor", + {{"type", "buffer"}, + {"sizes", {1, 3, 32, 32}}, + {"dtype", "float32"}}}}, + {{"tensor", + {{"type", "constant"}, + {"sizes", {16, 3, 3, 3}}, + {"layout", "height"}, + {"dtype", "float16"}}}}, + {{"tensor", + {{"type", "texture2d"}, + {"sizes", {16}}, + {"layout", "height"}, + {"dtype", "float32"}}}}, + {{"tensor", + {{"type", "buffer"}, + {"sizes", {60, 64, 40, 51}}, + {"layout", "4w4c"}, + {"dtype", "int8x4"}}}}, + {{"int_list", {2, 2}}}, + {{"int_list", {1, 1}}}, + {{"int_list", {1, 1}}}, + {{"bool", false}}, + {{"float", 0.5}}, + {{"string", "same"}}, + {{"int", 1}}, + {{"tensor", + {{"type", "texture3d"}, + {"sizes", {1, 16, 16, 16}}, + {"dtype", "float32"}, + {"access", "write"}}}}})}}; + report.update(measurement_report(1, true, {1}, {})); + TableOptions options; + options.inputs_max_chars = 0; + const std::string table = format_table(Report::array({report}), options); + EXPECT_NE( + table.find( + "b[1,3,32,32]f32.width, c[16,3,3,3]f16, t2[16]f32.height, b[60,64,40,51]i8x4.4w4c, [2,2], [1,1], [1,1], false, 0.5, \"same\", 1"), + std::string::npos); + EXPECT_EQ(table.find("c[16,3,3,3]f16."), std::string::npos); + EXPECT_EQ(table.find("t3[1,16,16,16]"), std::string::npos); +} + +TEST(BenchmarkReport, InputFloatsUseThreeSignificantFigures) { + Report report = { + {"operator_name", "op"}, + {"args", + Report::array( + {{{"float", 1.234567}}, + {{"float", -0.01234567}}, + {{"float", 9.999}}, + {{"float_list", {12345.67, 0.0000001234567, 0.0, 0.5}}}, + {{"int", 1234567}}, + {{"int_list", {12345, 67890}}}, + {{"string", "1.234567"}}, + false, + nullptr})}}; + report.update(measurement_report(1, true, {1}, {})); + const Report original = report; + const std::string expected = + "1.23, -0.0123, 10, [1.23e+04,1.23e-07,0,0.5], 1234567, [12345,67890], \"1.234567\", false, null"; + TableOptions options; + options.inputs_max_chars = 0; + EXPECT_NE( + format_table(Report::array({report}), options).find(expected), + std::string::npos); + const Report compact = Report::parse(format_condensed_report(report)); + EXPECT_EQ(compact.at("args_str"), expected); + EXPECT_EQ( + Report::parse(format_report(report)).at("args"), original.at("args")); + EXPECT_EQ(report, original); +} + +TEST(BenchmarkReport, ShaderInputFloatsUseThreeSignificantFigures) { + Report report = { + {"shader_name", "shader"}, + {"gwg_size", {{"value", {1, 1, 1}}}}, + {"lwg_size", {{"value", {1, 1, 1}}}}, + {"bindings", + {{"params", + {{"type", "uniform"}, + {"values", Report::array({{{"data", {1.234567, 1234567}}}})}}}}}, + {"push_constants", Report::array({{{"data", {-0.01234567, 9.999}}}})}}; + report.update(measurement_report(1, false, {1}, {})); + EXPECT_NE( + format_table(Report::array({report})) + .find("[1.23,1234567], [-0.0123,10]"), + std::string::npos); + const Report compact = Report::parse(format_condensed_report(report)); + EXPECT_EQ(compact.at("bindings"), report.at("bindings")); + EXPECT_EQ(compact.at("push_constants"), report.at("push_constants")); +} + +TEST(BenchmarkReport, TableSeparatesCasesButKeepsTheirShadersTogether) { + Report report = { + {"operator_name", "first"}, + {"args", Report::array()}, + {"shader_breakdown", + Report::array( + {{{"shader_name", "child"}, + {"gwg_size", {{"value", {1, 1, 1}}}}, + {"lwg_size", {{"value", {1, 1, 1}}}}, + {"gpu_timestamp_shader_us", {{"mean", 1}, {"p90", 1}}}}})}}; + report.update(measurement_report(1, true, {2}, {})); + Report second = report; + second["operator_name"] = "second"; + second.erase("shader_breakdown"); + std::istringstream stream(format_table(Report::array({report, second}))); + std::vector lines; + for (std::string line; std::getline(stream, line);) { + lines.push_back(line); + } + ASSERT_EQ(lines.size(), 6); + EXPECT_EQ(lines[1], std::string(lines[0].size(), '-')); + EXPECT_EQ(lines[4], lines[1]); + EXPECT_EQ(lines[2].find("first"), 0); + EXPECT_EQ(lines[3].find(" child"), 0); + EXPECT_EQ(lines[5].find("second"), 0); +} + +TEST(BenchmarkReport, TableLimitsTextWithoutChangingReports) { + Report report = { + {"operator_name", "operator_name_is_long"}, + {"args", Report::array({{{"string", "abcdefghijklmnopqrstuvwxyz"}}})}, + {"shader_breakdown", + Report::array( + {{{"shader_name", "shader_name_is_long"}, + {"gwg_size", {{"value", {1, 1, 1}}}}, + {"lwg_size", {{"value", {1, 1, 1}}}}, + {"gpu_timestamp_shader_us", {{"mean", 1}, {"p90", 1}}}}})}}; + report.update(measurement_report(1, true, {2}, {})); + const Report original = report; + TableOptions options; + options.name_max_chars = 12; + options.inputs_max_chars = 10; + const std::string limited = format_table(Report::array({report}), options); + EXPECT_NE(limited.find("operator_..."), std::string::npos); + EXPECT_NE(limited.find(" shader_..."), std::string::npos); + EXPECT_NE(limited.find("\"abcdef..."), std::string::npos); + EXPECT_EQ(limited.find("shader_name_is_long"), std::string::npos); + options.name_max_chars = 0; + options.inputs_max_chars = 0; + const std::string full = format_table(Report::array({report}), options); + EXPECT_NE(full.find("operator_name_is_long"), std::string::npos); + EXPECT_NE(full.find(" shader_name_is_long"), std::string::npos); + EXPECT_NE(full.find("\"abcdefghijklmnopqrstuvwxyz\""), std::string::npos); + options.name_max_chars = 3; + options.inputs_max_chars = 2; + const std::string tiny = format_table(Report::array({report}), options); + EXPECT_NE(tiny.find(" ."), std::string::npos); + EXPECT_NE(tiny.find(".."), std::string::npos); + EXPECT_EQ(report, original); +} + +TEST(BenchmarkReport, TableDisplaysOptionalValidationStatus) { + Report report = {{"operator_name", "op"}, {"args", Report::array()}}; + report.update(measurement_report(1, true, {2}, {})); + EXPECT_EQ( + format_table(Report::array({report})).find("Validation"), + std::string::npos); + Report passed = report; + passed["validation"] = {{"status", "PASS"}}; + Report failed = report; + failed["validation"] = {{"status", "FAIL"}}; + const std::string table = + format_table(Report::array({passed, failed, report})); + EXPECT_NE(table.find("Validation"), std::string::npos); + EXPECT_NE(table.find("PASS"), std::string::npos); + EXPECT_NE(table.find("FAIL"), std::string::npos); + EXPECT_NE(table.find("SKIP"), std::string::npos); +} + +TEST(BenchmarkReport, ReportsOmitLegacyCaseMetadata) { + const Json spec = { + {"operator", "aten.linear.default"}, + {"args", Json::array()}, + {"tensors", Json::object()}, + {"legacy_case", {{"op", "linear"}, {"set", "debug"}, {"index", 0}}}}; + EXPECT_FALSE(operator_report(spec, {}).contains("legacy_case")); + EXPECT_FALSE(error_report(spec, "test error").contains("legacy_case")); +} + +TEST(BenchmarkReport, ExplicitRouteMetadataSurvivesResultsAndErrors) { + Json spec = { + {"operator", "aten.convolution.default"}, + {"args", Json::array()}, + {"tensors", Json::object()}, + {"route", + {{"name", "im2col"}, + {"options", {{"intermediate_storage", "buffer"}}}}}}; + const Report expected = { + {"requested", spec.at("route")}, {"resolved", nullptr}}; + EXPECT_EQ(operator_report(spec, Json::object()).at("route"), expected); + EXPECT_EQ(error_report(spec, "Unsupported shape").at("route"), expected); + spec.erase("route"); + EXPECT_FALSE(operator_report(spec, Json::object()).contains("route")); + EXPECT_FALSE(error_report(spec, "Unsupported shape").contains("route")); +} + +TEST(BenchmarkReport, TableShowsRequestedAndResolvedRoutes) { + Report forced = { + {"operator_name", "forced"}, + {"args", Report::array()}, + {"route", + {{"requested", + {{"name", "im2col"}, + {"options", {{"intermediate_storage", "buffer"}}}}}, + {"resolved", + {{"name", "im2col"}, + {"options", {{"intermediate_storage", "buffer"}}}}}}}, + {"shader_breakdown", + Report::array( + {{{"shader_name", "child"}, + {"gwg_size", {{"value", {1, 1, 1}}}}, + {"lwg_size", {{"value", {1, 1, 1}}}}, + {"gpu_timestamp_shader_us", {{"mean", 1}, {"p90", 1}}}}})}}; + forced.update(measurement_report(1, true, {2}, {})); + Report automatic = forced; + automatic["operator_name"] = "automatic"; + automatic["route"] = { + {"requested", {{"name", "auto"}, {"options", Report::object()}}}, + {"resolved", {{"name", "depthwise"}, {"options", Report::object()}}}}; + automatic.erase("shader_breakdown"); + Report unknown = automatic; + unknown["operator_name"] = "unknown"; + unknown["route"]["resolved"] = nullptr; + Report omitted = unknown; + omitted["operator_name"] = "omitted"; + omitted.erase("route"); + const std::string table = + format_table(Report::array({forced, automatic, unknown, omitted})); + EXPECT_LT(table.find("Shader / operator"), table.find("Route")); + EXPECT_LT(table.find("Route"), table.find("GWG")); + EXPECT_NE(table.find("im2col[buffer]"), std::string::npos); + EXPECT_NE(table.find("auto->depthwise"), std::string::npos); + std::istringstream stream(table); + std::vector lines; + for (std::string line; std::getline(stream, line);) { + lines.push_back(line); + } + ASSERT_EQ(lines.size(), 10); + EXPECT_EQ(lines[1], std::string(lines[0].size(), '-')); + std::istringstream child(lines[3]); + std::string shader_name, route, gwg; + child >> shader_name >> route >> gwg; + EXPECT_EQ(shader_name, "child"); + EXPECT_EQ(route, "-"); + EXPECT_EQ(gwg, "[1,1,1]"); + EXPECT_NE(lines[7].find("auto"), std::string::npos); + EXPECT_EQ(lines[7].find("auto->"), std::string::npos); +} + +TEST(BenchmarkReport, TableKeepsRouteOnUnsupportedCaseErrors) { + const Report error = error_report( + {{"operator", "aten.convolution.default"}, + {"route", + {{"name", "im2col"}, + {"options", {{"intermediate_storage", "texture3d"}}}}}}, + "Unsupported shape"); + const std::string table = format_table(Report::array({error})); + EXPECT_NE(table.find("im2col[texture3d]"), std::string::npos); + EXPECT_NE(table.find("ERROR"), std::string::npos); + EXPECT_NE(table.find("Unsupported shape"), std::string::npos); +} + +TEST(BenchmarkReport, TableRendersSkippedCasesWithoutTimingMeasurements) { + const Report skipped = { + {"case_name", "forced convolution"}, + {"operator_name", "aten.convolution.default"}, + {"sweep_index", 2}, + {"status", "SKIP"}, + {"skip_reason", "Unsupported\nshape"}, + {"route", + {{"requested", + {{"name", "im2col"}, + {"options", {{"intermediate_storage", "buffer"}}}}}, + {"resolved", nullptr}}}, + {"args", + Report::array( + {{{"tensor", + {{"sizes", {1, 3, 8, 8}}, + {"dtype", "float32"}, + {"type", "buffer"}}}}, + {{"int_list", {2, 2}}}, + {{"tensor", + {{"sizes", {1, 16, 4, 4}}, + {"dtype", "float32"}, + {"type", "buffer"}, + {"access", "write"}}}}})}, + {"throughput", {{"memory_bytes", nullptr}}}}; + TableOptions options; + options.name_max_chars = 0; + const std::string table = format_table(Report::array({skipped}), options); + EXPECT_NE( + table.find("aten.convolution.default [2] (SKIP)"), std::string::npos); + EXPECT_NE(table.find("im2col[buffer]"), std::string::npos); + EXPECT_NE(table.find("b[1,3,8,8]f32.width, [2,2]"), std::string::npos); + EXPECT_EQ(table.find("b[1,16,4,4]"), std::string::npos); + EXPECT_NE(table.find("N/A"), std::string::npos); + EXPECT_NE( + table.find("SKIP forced convolution [2]: Unsupported shape"), + std::string::npos); + Report without_args = skipped; + without_args.erase("args"); + without_args["validation"] = {{"status", "SKIP"}}; + EXPECT_NE( + format_table(Report::array({without_args})).find("Validation"), + std::string::npos); +} + +TEST(BenchmarkReport, TableDisplaysRequestedGpuThroughputColumns) { + Report report = {{"operator_name", "op"}, {"args", Report::array()}}; + report.update(measurement_report(1, true, {2}, {})); + const std::string disabled = format_table(Report::array({report})); + EXPECT_EQ(disabled.find("GPU GiB/s"), std::string::npos); + EXPECT_EQ(disabled.find("GPU GFLOPS"), std::string::npos); + + report["throughput"] = { + {"memory_bytes", 4096}, + {"gpu_timestamp", {{"memory_gib_per_s", {{"mean", 4.125}}}}}}; + const std::string memory_only = format_table(Report::array({report})); + EXPECT_NE(memory_only.find("GPU GiB/s"), std::string::npos); + EXPECT_NE(memory_only.find("4.125"), std::string::npos); + EXPECT_EQ(memory_only.find("GPU GFLOPS"), std::string::npos); + + report["throughput"] = { + {"compute_flops", 8192}, + {"gpu_timestamp", {{"compute_gflops", {{"mean", 8.625}}}}}}; + const std::string compute_only = format_table(Report::array({report})); + EXPECT_EQ(compute_only.find("GPU GiB/s"), std::string::npos); + EXPECT_NE(compute_only.find("GPU GFLOPS"), std::string::npos); + EXPECT_NE(compute_only.find("8.625"), std::string::npos); + + report["validation"] = {{"status", "PASS"}}; + report["throughput"]["memory_bytes"] = 0; + report["throughput"]["gpu_timestamp"]["memory_gib_per_s"] = {{"mean", 0}}; + const std::string both = format_table(Report::array({report})); + EXPECT_LT(both.find("Validation"), both.find("GPU GiB/s")); + EXPECT_LT(both.find("GPU GiB/s"), both.find("GPU GFLOPS")); + EXPECT_NE(both.find("0.000"), std::string::npos); + EXPECT_NE(both.find("8.625"), std::string::npos); +} + +TEST(BenchmarkReport, TableIncludesShaderThroughputBelowOperator) { + Report report = { + {"operator_name", "op"}, + {"args", Report::array()}, + {"throughput", + {{"memory_bytes", 4096}, + {"compute_flops", 8192}, + {"gpu_timestamp", + {{"memory_gib_per_s", {{"mean", 4.125}}}, + {"compute_gflops", {{"mean", 8.625}}}}}}}, + {"shader_breakdown", + Report::array( + {{{"shader_name", "child"}, + {"gwg_size", {{"value", {1, 1, 1}}}}, + {"lwg_size", {{"value", {1, 1, 1}}}}, + {"gpu_timestamp_shader_us", {{"mean", 1}, {"p90", 1}}}, + {"throughput", + {{"memory_bytes", 1024}, + {"compute_flops", 2048}, + {"gpu_timestamp", + {{"memory_gib_per_s", {{"mean", 1.125}}}, + {"compute_gflops", {{"mean", 2.625}}}}}}}}})}}; + report.update(measurement_report(1, true, {2}, {})); + std::istringstream stream(format_table(Report::array({report}))); + std::vector lines; + for (std::string line; std::getline(stream, line);) { + lines.push_back(line); + } + ASSERT_EQ(lines.size(), 4); + EXPECT_EQ(lines[1], std::string(lines[0].size(), '-')); + EXPECT_NE(lines[2].find("4.125"), std::string::npos); + EXPECT_NE(lines[2].find("8.625"), std::string::npos); + EXPECT_EQ(lines[3].find(" child"), 0); + EXPECT_NE(lines[3].find("1.125"), std::string::npos); + EXPECT_NE(lines[3].find("2.625"), std::string::npos); +} + +TEST( + BenchmarkReport, + TableThroughputUsesNaForMissingEstimatesAndGpuTimestamps) { + Report report = { + {"operator_name", "no_timestamps"}, {"args", Report::array()}}; + report.update(measurement_report(1, true, {}, {2})); + report["throughput"] = { + {"memory_bytes", 4096}, + {"compute_flops", nullptr}, + {"gpu_timestamp", + {{"memory_gib_per_s", nullptr}, {"compute_gflops", nullptr}}}, + {"wall_time", {{"memory_gib_per_s", {111.625}}}}, + {"errors", {{"compute", "No estimate"}}}}; + Report without_estimates = report; + without_estimates["operator_name"] = "without_estimates"; + without_estimates.erase("throughput"); + const Report error = error_report({{"operator", "error_case"}}, "Bad input"); + std::istringstream stream( + format_table(Report::array({report, without_estimates, error}))); + std::vector lines; + for (std::string line; std::getline(stream, line);) { + lines.push_back(line); + } + ASSERT_EQ(lines.size(), 8); + EXPECT_NE(lines[0].find("GPU GiB/s"), std::string::npos); + EXPECT_NE(lines[0].find("GPU GFLOPS"), std::string::npos); + for (size_t i : {2u, 4u, 6u}) { + EXPECT_EQ(lines[i].substr(lines[i].size() - 3), "N/A"); + EXPECT_EQ(lines[i].find("111.625"), std::string::npos); + } + EXPECT_NE(lines[7].find("ERROR error_case: Bad input"), std::string::npos); +} + +TEST(BenchmarkReport, OperatorTensorMetadataIsInlineWithoutValues) { + const Json arg_set = Json::array( + {{{"tensor", + {{"name", "x"}, + {"sizes", {2}}, + {"dtype", "float16"}, + {"data", {1, 2}}}}}, + {{"tensor", + {{"name", "w"}, + {"sizes", {2}}, + {"storage", "constant"}, + {"fill", 0.5}}}}, + nullptr, + {{"int", 32}}, + {{"tensor", {{"ref", "x"}}}}, + {{"tensor", {{"ref", "x"}, {"access", "write"}}}}}); + const Json spec = parse_configs({{"operator", "aten.linear.default"}, + {"args_sweep", Json::array({arg_set})}}) + .at(0); + const Report report = + operator_report(spec, {{"x", "float32"}, {"w", "float32"}}); + EXPECT_FALSE(report.contains("tensors")); + EXPECT_FALSE(report.contains("inputs")); + EXPECT_FALSE(report.contains("outputs")); + EXPECT_EQ(report.at("sweep_index"), 0); + const auto& args = report.at("args"); + ASSERT_EQ(args.size(), 6); + EXPECT_EQ(args.at(0).at("tensor").at("name"), "x"); + EXPECT_EQ(args.at(0).at("tensor").at("dtype"), "float32"); + EXPECT_EQ(args.at(0).at("tensor").at("requested_dtype"), "float16"); + EXPECT_EQ(args.at(0).at("tensor").at("access"), "read"); + EXPECT_EQ(args.at(1).at("tensor").at("type"), "constant"); + EXPECT_FALSE(args.at(1).at("tensor").contains("constant")); + EXPECT_TRUE(args.at(2).is_null()); + EXPECT_EQ(args.at(3), (Report{{"int", 32}})); + EXPECT_EQ(args.at(4).at("tensor"), args.at(0).at("tensor")); + Report output = args.at(0).at("tensor"); + output["access"] = "write"; + EXPECT_EQ(args.at(5).at("tensor"), output); + EXPECT_EQ(report.dump().find("\"data\""), std::string::npos); + EXPECT_EQ(report.dump().find("\"fill\""), std::string::npos); +} + +TEST(BenchmarkReport, InferredTensorRolesAndResolvedVariablesAreVisible) { + const Json args = Json::array( + {{{"tensor", {{"name", "x"}, {"sizes", {2}}, {"dtype", "v_DTYPE"}}}}, + {{"tensor", {{"name", "y"}, {"sizes", {2}}, {"dtype", "v_DTYPE"}}}}}); + const auto cases = parse_configs( + {{"operator", "aten.relu.default"}, + {"sweep_variables", {{"v_DTYPE", {"float32", "float16"}}}}, + {"args_sweep", Json::array({args})}}); + const Report report = operator_report( + cases[1], + {{"x", "float16"}, {"y", "float16"}}, + {{"x", "read"}, {"y", "write"}}); + EXPECT_EQ(report.at("sweep_index"), 1); + EXPECT_EQ(report.at("args_sweep_index"), 0); + EXPECT_EQ(report.at("sweep_variables").at("v_DTYPE"), "float16"); + EXPECT_EQ(report.at("args")[0].at("tensor").at("access"), "read"); + EXPECT_EQ(report.at("args")[1].at("tensor").at("access"), "write"); +} + +TEST(BenchmarkReport, SweepIndexesIdentifyResultsAndErrors) { + Report result = { + {"case_name", "linear"}, + {"operator_name", "aten.linear.default"}, + {"sweep_index", 1}, + {"args", Report::array()}}; + result.update(measurement_report(1, true, {5}, {6})); + const Report error = error_report( + {{"name", "linear"}, + {"operator", "aten.linear.default"}, + {"sweep_index", 0}}, + "missing argument"); + EXPECT_EQ(error.at("sweep_index"), 0); + const std::string table = format_table(Report::array({error, result})); + EXPECT_NE(table.find("aten.linear.default [0]"), std::string::npos); + EXPECT_NE(table.find("aten.linear.default [1]"), std::string::npos); + EXPECT_NE( + table.find("ERROR linear [0]: missing argument"), std::string::npos); +} + +TEST(BenchmarkReport, TablePreservesCaseErrors) { + const Report report = error_report( + {{"shader", "missing"}, {"name", "bad\ncase"}}, "Unknown\tshader"); + const std::string table = format_table(Report::array({report})); + EXPECT_NE(table.find("ERROR bad case: Unknown shader"), std::string::npos); + EXPECT_NE(table.find("missing"), std::string::npos); +} + +TEST(BenchmarkReport, TableShowsObservedClockRangeAcrossAllPasses) { + Report report = { + {"operator_name", "op"}, + {"args", Report::array()}, + {"gpu_batch_observed_gpu_clock_freq_MHz", {{300, 400}, {nullptr, 350}}}, + {"wall_time_batch_observed_gpu_clock_freq_MHz", {{200, 250}, {0, 100}}}, + {"shader_profile_batch_observed_gpu_clock_freq_MHz", + {{600, 700}, {500, nullptr}}}, + {"shader_breakdown", + Report::array( + {{{"shader_name", "child"}, + {"gwg_size", {{"value", {1, 1, 1}}}}, + {"lwg_size", {{"value", {1, 1, 1}}}}, + {"gpu_timestamp_shader_us", {{"mean", 1}, {"p90", 2}}}}})}}; + report.update(measurement_report(1, true, {2, 3, 4}, {})); + Report missing = {{"operator_name", "missing"}, {"args", Report::array()}}; + missing.update(measurement_report(1, true, {2}, {})); + const std::string table = format_table(Report::array({report, missing})); + EXPECT_NE(table.find("Min GPU MHz"), std::string::npos); + EXPECT_NE(table.find("Max GPU MHz"), std::string::npos); + std::istringstream lines(table); + for (std::string line; std::getline(lines, line);) { + std::istringstream words(line); + std::vector fields; + for (std::string word; words >> word;) + fields.push_back(word); + if (fields.empty()) + continue; + if (fields[0] == "op") { + EXPECT_EQ(fields[fields.size() - 2], "0"); + EXPECT_EQ(fields.back(), "700"); + } else if (fields[0] == "child") { + EXPECT_EQ(fields[fields.size() - 2], "500"); + EXPECT_EQ(fields.back(), "700"); + } else if (fields[0] == "missing") { + EXPECT_EQ(fields[fields.size() - 2], "N/A"); + EXPECT_EQ(fields.back(), "N/A"); + } + } +} + +TEST(BenchmarkReport, TableHandlesUnavailableAndFractionalClockReadings) { + Report report = { + {"shader_name", "copy"}, + {"gwg_size", {{"value", {1, 1, 1}}}}, + {"lwg_size", {{"value", {1, 1, 1}}}}, + {"bindings", Report::object()}, + {"gpu_batch_observed_gpu_clock_freq_MHz", {{nullptr, nullptr}}}}; + report.update(measurement_report(1, false, {}, {10, 20})); + std::string table = format_table(Report::array({report})); + EXPECT_NE(table.find("Min GPU MHz"), std::string::npos); + EXPECT_EQ(table.substr(table.size() - 3), "N/A"); + report["wall_time_batch_observed_gpu_clock_freq_MHz"] = {{333.125, 444.75}}; + table = format_table(Report::array({report})); + EXPECT_NE(table.find("333.125"), std::string::npos); + EXPECT_NE(table.find("444.75"), std::string::npos); +} + +TEST(BenchmarkReport, GpuClockPairsStayOnOneLine) { + Report report = { + {"shader_name", "copy"}, + {"gwg_size", {{"value", {1, 1, 1}}}}, + {"lwg_size", {{"value", {1, 1, 1}}}}, + {"bindings", Report::object()}}; + report.update(measurement_report(1, false, {2}, {3})); + for (const char* field : + {"gpu_batch_observed_gpu_clock_freq_MHz", + "wall_time_batch_observed_gpu_clock_freq_MHz", + "shader_profile_batch_observed_gpu_clock_freq_MHz"}) { + Report clocked = report; + clocked[field] = {{903, 950}, {nullptr, 800}}; + const std::string text = format_report(clocked); + EXPECT_NE(text.find("[903, 950]"), std::string::npos); + EXPECT_NE(text.find("[null, 800]"), std::string::npos); + EXPECT_EQ(Report::parse(text), clocked); + } +} + +TEST(BenchmarkReport, DimensionArraysStayOnOneLine) { + const Report report = Report::array( + {{{"gwg_size", {{"method", "explicit"}, {"value", {8, 8, 1}}}}, + {"lwg_size", {{"method", "default_pick_lwg"}, {"value", {4, 4, 1}}}}, + {"bindings", + {{"input", {{"type", "texture3d"}, {"sizes", {1, 3, 224, 224}}}}}}, + {"wall_time_batch_us", {12.5, 13.5}}}}); + const std::string text = format_report(report); + EXPECT_NE(text.find("\"value\": [8, 8, 1]"), std::string::npos); + EXPECT_NE(text.find("\"value\": [4, 4, 1]"), std::string::npos); + EXPECT_NE(text.find("\"sizes\": [1, 3, 224, 224]"), std::string::npos); + EXPECT_NE(text.find("\"wall_time_batch_us\": [\n"), std::string::npos); + EXPECT_EQ(Report::parse(text), report); +} + +TEST(BenchmarkReport, FormattingPreservesEscapedStringsAndFieldOrder) { + const Report report = { + {"case_name", "quoted \"name\"\nline"}, + {"labels", {"[", "\"", "\\", "\n"}}, + {"tensors", {{"weight", {{"sizes", {8, 32}}}}}}, + {"empty_object", Report::object()}, + {"empty_array", Report::array()}}; + const std::string text = format_report(report); + EXPECT_EQ(Report::parse(text), report); + EXPECT_NE(text.find("\"sizes\": [8, 32]"), std::string::npos); +} + +TEST(BenchmarkReport, CondensedJsonUsesInputsAndSummarizesBatchTimings) { + Report report = { + {"operator_name", "aten.linear.default"}, + {"args", + Report::array( + {{{"tensor", + {{"type", "buffer"}, + {"sizes", {4, 32}}, + {"dtype", "float32"}, + {"layout", "width"}, + {"access", "read"}}}}, + {{"tensor", + {{"type", "constant"}, + {"sizes", {16, 32}}, + {"dtype", "float32"}, + {"layout", "width"}}}}, + {{"int_list", {1, 2}}}, + {{"tensor", + {{"type", "buffer"}, + {"sizes", {4, 16}}, + {"dtype", "float32"}, + {"access", "write"}}}}})}, + {"benchmark_runs", 4}, + {"shader_breakdown", + Report::array( + {{{"shader_name", "linear"}, + {"gpu_timestamp_shader_us", latency_statistics({1, 2, 3})}}})}, + {"validation", {{"status", "PASS"}}}}; + report.update( + measurement_report(10, true, {60, 20, 90, 30}, {80, 40, 20, 60})); + const Report original = report; + const Report condensed = Report::parse(format_condensed_report(report)); + EXPECT_FALSE(condensed.contains("args")); + EXPECT_EQ(condensed.at("args_str"), "b[4,32]f32.width, c[16,32]f32, [1,2]"); + EXPECT_EQ( + condensed.at("gpu_timestamp_operator_us"), + (Report{{"min", 2}, {"median", 4.5}, {"max", 9}})); + EXPECT_EQ( + condensed.at("gpu_timestamp_batch_us"), + (Report{{"min", 20}, {"median", 45}, {"max", 90}})); + EXPECT_EQ( + condensed.at("wall_time_operator_us"), + (Report{{"min", 2}, {"median", 5}, {"max", 8}})); + EXPECT_EQ(condensed.at("shader_breakdown"), report.at("shader_breakdown")); + EXPECT_EQ(condensed.at("validation"), report.at("validation")); + EXPECT_EQ(report, original); +} + +TEST(BenchmarkReport, CondensedJsonSummarizesPassClockRanges) { + const Report report = { + {"gpu_batch_observed_gpu_clock_freq_MHz", {{903, 800}, {nullptr, 950}}}, + {"wall_time_batch_observed_gpu_clock_freq_MHz", {{0, 700}, {600, 650}}}, + {"shader_profile_batch_observed_gpu_clock_freq_MHz", + {{nullptr, nullptr}}}, + {"gpu_timestamp_operator_us", Report::array()}, + {"gpu_timestamp_batch_us", Report::array()}, + {"wall_time_batch_us", {3, 1, 2}}}; + const Report condensed = Report::parse(format_condensed_report(report)); + EXPECT_EQ( + condensed.at("gpu_batch_observed_gpu_clock_freq_MHz"), + (Report{{"min", 800}, {"max", 950}})); + EXPECT_EQ( + condensed.at("wall_time_batch_observed_gpu_clock_freq_MHz"), + (Report{{"min", 0}, {"max", 700}})); + EXPECT_TRUE(condensed.at("shader_profile_batch_observed_gpu_clock_freq_MHz") + .is_null()); + EXPECT_TRUE(condensed.at("gpu_timestamp_operator_us").is_null()); + EXPECT_TRUE(condensed.at("gpu_timestamp_batch_us").is_null()); + EXPECT_EQ( + condensed.at("wall_time_batch_us"), + (Report{{"min", 1}, {"median", 2}, {"max", 3}})); +} + +TEST(BenchmarkReport, CondensedJsonPreservesShaderStatisticsAndCaseGroups) { + Report report = { + {"shader_name", "copy"}, + {"bindings", {{"input", {{"sizes", {8}}, {"dtype", "float32"}}}}}}; + report.update(measurement_report(10, false, {10, 20, 30}, {30, 10, 20})); + const Report error = error_report({{"operator", "bad"}}, "test error"); + const Report condensed = + Report::parse(format_condensed_report(Report::array({report, error}))); + ASSERT_EQ(condensed.size(), 2); + EXPECT_EQ(condensed[0].at("bindings"), report.at("bindings")); + EXPECT_EQ( + condensed[0].at("gpu_timestamp_shader_us"), + report.at("gpu_timestamp_shader_us")); + EXPECT_EQ( + condensed[0].at("wall_time_shader_us"), + (Report{{"min", 1}, {"median", 2}, {"max", 3}})); + EXPECT_EQ(condensed[1].at("error"), "test error"); +} + +TEST(BenchmarkReport, StatisticsUseInterpolatedPercentiles) { + const Report stats = latency_statistics({5, 1, 4, 2, 3}); + ASSERT_TRUE(stats.is_object()); + EXPECT_EQ(stats.at("min"), 1); + EXPECT_EQ(stats.at("max"), 5); + EXPECT_EQ(stats.at("mean"), 3); + EXPECT_EQ(stats.at("p50"), 3); + EXPECT_NEAR(stats.at("p90").get(), 4.6, 1e-12); + EXPECT_NEAR(stats.at("p95").get(), 4.8, 1e-12); + EXPECT_TRUE(latency_statistics({}).is_null()); +} + +TEST(BenchmarkReport, PerInvocationSamplesAreDividedByRepetitions) { + const Report report = + measurement_report(10, false, {10, 20, 30}, {20, 40, 60}); + ASSERT_TRUE(report.contains("gpu_timestamp_shader_us")); + EXPECT_EQ(report.at("gpu_timestamp_shader_us").at("mean"), 2); + EXPECT_EQ(report.at("gpu_timestamp_batch_us"), Report::array({10, 20, 30})); + EXPECT_EQ(report.at("wall_time_shader_us"), Report::array({2, 4, 6})); + EXPECT_EQ(report.at("wall_time_batch_us"), Report::array({20, 40, 60})); + EXPECT_THROW(measurement_report(0, false, {}, {1}), std::invalid_argument); +} + +TEST(BenchmarkReport, OperatorGpuSamplesPreserveRunOrderAndBatchAverages) { + Report report = {{"operator_name", "op"}, {"args", Report::array()}}; + report.update(measurement_report(10, true, {30, 10, 20}, {60, 20, 40})); + EXPECT_EQ(report.at("gpu_timestamp_batch_us"), Report::array({30, 10, 20})); + EXPECT_EQ(report.at("gpu_timestamp_operator_us"), Report::array({3, 1, 2})); + EXPECT_EQ(report.at("wall_time_operator_us"), Report::array({6, 2, 4})); + const std::string table = format_table(Report::array({report})); + EXPECT_NE(table.find("2.000"), std::string::npos); + EXPECT_NE(table.find("2.800"), std::string::npos); +} + +TEST(BenchmarkReport, UnavailableTimestampsAndOperatorsUseExplicitLabels) { + const Report report = measurement_report(2, true, {}, {4, 8}); + ASSERT_TRUE(report.contains("gpu_timestamp_operator_us")); + EXPECT_EQ(report.at("gpu_timestamp_operator_us"), Report::array()); + EXPECT_EQ(report.at("gpu_timestamp_batch_us"), Report::array()); + EXPECT_EQ(report.at("wall_time_operator_us"), Report::array({2, 4})); + EXPECT_FALSE(report.contains("gpu_timestamp_shader_us")); +} + +TEST(BenchmarkReport, ShaderDetailsHaveStableOrderAndNoTensorValues) { + const Json spec = + parse_configs( + {{"name", "copy"}, + {"shader", "buffer_to_buffer_float"}, + {"gwg_size", {8, 1, 1}}, + {"lwg_size", "default_pick_lwg"}, + {"bindings", + Json::array( + {{{"buffer", + {{"name", "output"}, {"sizes", {8}}, {"access", "write"}}}}, + {{"buffer", + {{"name", "input"}, + {"sizes", {8}}, + {"data", {1, 2, 3, 4, 5, 6, 7, 8}}}}}, + {{"meta_ubo", "output"}, {"field", "numel"}}})}, + {"outputs", {"output"}}, + {"dump_outputs", true}}) + .at(0); + const Report report = shader_report( + spec, + {8, 1, 1}, + {8, 1, 1}, + {{"input", "float32"}, {"output", "float32"}}, + Report::object()); + std::vector keys; + for (const auto& item : report.items()) { + keys.push_back(item.key()); + } + EXPECT_EQ( + keys, + (std::vector{ + "case_name", + "shader_name", + "gwg_size", + "lwg_size", + "bindings", + "specialization_constants"})); + ASSERT_TRUE(report.contains("bindings")); + EXPECT_EQ(report.at("gwg_size").at("method"), "explicit"); + EXPECT_EQ(report.at("lwg_size").at("method"), "default_pick_lwg"); + EXPECT_EQ(report.at("bindings").at("input").at("type"), "buffer"); + EXPECT_EQ(report.at("bindings").at("input").at("binding_index"), 1); + EXPECT_FALSE(report.at("bindings").at("input").contains("data")); + EXPECT_FALSE(report.contains("outputs")); +} + +TEST(BenchmarkReport, SharedBindingsAndPrecisionPromotionRemainVisible) { + const Json spec = + parse_configs({{"shader", "buffer_to_buffer_float"}, + {"gwg_size", {8, 1, 1}}, + {"lwg_size", {8, 1, 1}}, + {"bindings", + Json::array( + {{{"buffer", + {{"name", "shared"}, + {"sizes", {8}}, + {"dtype", "float16"}, + {"access", "write"}}}}, + {{"buffer", {{"ref", "shared"}}}}, + {{"meta_ubo", "shared"}, {"field", "numel"}}})}}) + .at(0); + const Report report = shader_report( + spec, {8, 1, 1}, {8, 1, 1}, {{"shared", "float32"}}, Report::object()); + const auto& bindings = report.at("bindings"); + ASSERT_EQ(bindings.size(), 3); + EXPECT_EQ(bindings.at("shared").at("dtype"), "float32"); + EXPECT_EQ(bindings.at("shared").at("requested_dtype"), "float16"); + std::vector indices; + for (const auto& info : bindings) { + indices.push_back(info.at("binding_index")); + if (info.at("binding_index") == 1) { + EXPECT_EQ(info.at("ref"), "shared"); + } + } + EXPECT_EQ(indices, (std::vector{0, 1, 2})); +} + +} // namespace +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/test/route_registry_test.cpp b/backends/vulkan/test/ops/benchmark_v2/framework/test/route_registry_test.cpp new file mode 100644 index 00000000000..ae9878aa76f --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/test/route_registry_test.cpp @@ -0,0 +1,111 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include + +namespace vkcompute::shader_profiling { +namespace { + +Json first_builder( + ComputeGraph&, + const std::vector&, + const Json& options) { + return {{"name", "first"}, {"options", options}}; +} +Json second_builder( + ComputeGraph&, + const std::vector&, + const Json& options) { + return {{"name", "second"}, {"options", options}}; +} +Json inspect_builder( + ComputeGraph&, + const std::vector&, + size_t, + size_t) { + return {{"name", "first"}, {"options", Json::object()}}; +} + +REGISTER_OPERATOR_ROUTES { + REGISTER_OPERATOR_ROUTE(test_operator, candidate, first_builder); + REGISTER_ROUTE_INSPECTOR(test_operator, inspect_builder); +} + +TEST(BenchmarkRouteRegistry, RoutesAreScopedToOperators) { + RouteRegistry registry; + registry.register_operator_route("op_a", "direct", first_builder); + registry.register_operator_route("op_b", "direct", second_builder); + registry.register_operator_route("op_a", "other", second_builder); + EXPECT_EQ(registry.operator_route("op_a", "direct"), first_builder); + EXPECT_EQ(registry.operator_route("op_b", "direct"), second_builder); + EXPECT_EQ(registry.operator_route("op_a", "other"), second_builder); +} + +TEST(BenchmarkRouteRegistry, BuildersCanBeSharedAcrossRoutes) { + RouteRegistry registry; + registry.register_operator_route("op_a", "direct", first_builder); + registry.register_operator_route("op_a", "other", first_builder); + registry.register_operator_route("op_b", "direct", first_builder); + EXPECT_EQ(registry.operator_route("op_a", "direct"), first_builder); + EXPECT_EQ(registry.operator_route("op_a", "other"), first_builder); + EXPECT_EQ(registry.operator_route("op_b", "direct"), first_builder); +} + +TEST(BenchmarkRouteRegistry, DuplicateRoutesPreserveOriginals) { + RouteRegistry registry; + registry.register_operator_route("op", "direct", first_builder); + EXPECT_THROW( + registry.register_operator_route("op", "direct", second_builder), + std::invalid_argument); + EXPECT_EQ(registry.operator_route("op", "direct"), first_builder); +} + +TEST(BenchmarkRouteRegistry, InspectorsAreOptionalAndUnique) { + RouteRegistry registry; + EXPECT_EQ(registry.operator_inspector("op"), nullptr); + registry.register_operator_inspector("op", inspect_builder); + EXPECT_EQ(registry.operator_inspector("op"), inspect_builder); + EXPECT_THROW( + registry.register_operator_inspector("op", inspect_builder), + std::invalid_argument); +} + +TEST(BenchmarkRouteRegistry, RejectsMissingAndInvalidRegistrations) { + RouteRegistry registry; + EXPECT_THROW(registry.operator_route("op", "missing"), std::invalid_argument); + registry.register_operator_route("op", "direct", first_builder); + EXPECT_THROW(registry.operator_route("op", "missing"), std::invalid_argument); + EXPECT_THROW( + registry.register_operator_route("", "direct", first_builder), + std::invalid_argument); + EXPECT_THROW( + registry.register_operator_route("op", "", first_builder), + std::invalid_argument); + EXPECT_THROW( + registry.register_operator_route("op", "direct", nullptr), + std::invalid_argument); + EXPECT_THROW( + registry.register_operator_route("op", "auto", first_builder), + std::invalid_argument); + EXPECT_THROW( + registry.register_operator_inspector("", inspect_builder), + std::invalid_argument); + EXPECT_THROW( + registry.register_operator_inspector("op", nullptr), + std::invalid_argument); +} + +TEST(BenchmarkRouteRegistry, StaticRegistrationsAreAvailable) { + const auto& registry = route_registry(); + EXPECT_EQ( + registry.operator_route("test_operator", "candidate"), first_builder); + EXPECT_EQ(registry.operator_inspector("test_operator"), inspect_builder); +} + +} // namespace +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/test/throughput_registry_test.cpp b/backends/vulkan/test/ops/benchmark_v2/framework/test/throughput_registry_test.cpp new file mode 100644 index 00000000000..de8e1336943 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/test/throughput_registry_test.cpp @@ -0,0 +1,510 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include + +#include +#include + +namespace vkcompute::shader_profiling { +namespace { + +double memory_count(const Json&) { + return 40; +} +double compute_count(const Json&) { + return 20; +} +double negative_count(const Json&) { + return -1; +} +double infinite_count(const Json&) { + return std::numeric_limits::infinity(); +} +double nan_count(const Json&) { + return std::numeric_limits::quiet_NaN(); +} +double throwing_count(const Json&) { + throw std::invalid_argument("bad work shape"); +} + +REGISTER_THROUGHPUT_FUNCTIONS { + REGISTER_OPERATOR_MEMORY_TRAFFIC(test_counter_case, memory_count); + REGISTER_OPERATOR_COMPUTE_FLOPS(test_counter_case, compute_count); + REGISTER_SHADER_MEMORY(test_counter_case, compute_count); + REGISTER_SHADER_COMPUTE(test_counter_case, memory_count); + REGISTER_SHADER_MEMORY(test_negative, negative_count); + REGISTER_SHADER_MEMORY(test_infinite, infinite_count); + REGISTER_SHADER_MEMORY(test_nan, nan_count); + REGISTER_SHADER_MEMORY(test_throwing, throwing_count); + REGISTER_SHADER_COMPUTE(test_throwing, compute_count); + REGISTER_SHADER_MEMORY_TRAFFIC(test_prefix, memory_count); + REGISTER_SHADER_COMPUTE_FLOPS(test_prefix, compute_count); +} + +Json tensor( + const Json& sizes, + const std::string& dtype = "float32", + const std::string& access = "read", + const std::string& type = "buffer") { + return { + {"sizes", sizes}, {"dtype", dtype}, {"access", access}, {"type", type}}; +} + +Json linear_case() { + return { + {"operator_name", "aten.linear.default"}, + {"args", + {{{"tensor", tensor({2, 3})}}, + {{"tensor", tensor({4, 3}, "float32", "read", "constant")}}, + {{"tensor", tensor({4}, "float32", "read", "constant")}}, + {{"tensor", tensor({2, 4}, "float32", "write")}}}}}; +} + +TEST(BenchmarkThroughputRegistry, OperatorAndShaderCountersAreIndependent) { + ThroughputRegistry registry; + registry.register_operator_memory("same", memory_count); + registry.register_operator_compute("same", compute_count); + registry.register_shader_memory("same", compute_count); + registry.register_shader_compute("same", memory_count); + EXPECT_EQ(registry.operator_memory("same"), memory_count); + EXPECT_EQ(registry.operator_compute("same"), compute_count); + EXPECT_EQ(registry.shader_memory("same"), compute_count); + EXPECT_EQ(registry.shader_compute("same"), memory_count); +} + +TEST(BenchmarkThroughputRegistry, ShaderPrefixesMatchOnlyStartsOfNames) { + ThroughputRegistry registry; + registry.register_shader_memory_prefix("conv2d", memory_count); + registry.register_shader_compute_prefix("conv2d", compute_count); + for (const char* name : + {"conv2d", "conv2d_float", "conv2d_gemm_texture3d_half"}) { + EXPECT_EQ(registry.shader_memory(name), memory_count); + EXPECT_EQ(registry.shader_compute(name), compute_count); + } + for (const char* name : {"q8ta_conv2d_float", "xconv2d", "Conv2d_float"}) { + EXPECT_THROW(registry.shader_memory(name), std::invalid_argument); + EXPECT_THROW(registry.shader_compute(name), std::invalid_argument); + } + EXPECT_THROW(registry.operator_memory("conv2d_float"), std::invalid_argument); +} + +TEST( + BenchmarkThroughputRegistry, + ExactShaderNamesOverrideLongestMatchingPrefix) { + for (bool broad_first : {false, true}) { + ThroughputRegistry registry; + for (const char* prefix : broad_first + ? std::vector{"conv2d", "conv2d_im2col"} + : std::vector{"conv2d_im2col", "conv2d"}) { + const auto function = + std::string(prefix) == "conv2d" ? memory_count : compute_count; + registry.register_shader_memory_prefix(prefix, function); + registry.register_shader_compute_prefix(prefix, function); + } + registry.register_shader_memory( + "conv2d_im2col_buffer_float", negative_count); + registry.register_shader_compute( + "conv2d_im2col_buffer_float", negative_count); + EXPECT_EQ(registry.shader_memory("conv2d_float"), memory_count); + EXPECT_EQ(registry.shader_compute("conv2d_float"), memory_count); + EXPECT_EQ( + registry.shader_memory("conv2d_im2col_texture3d_half"), compute_count); + EXPECT_EQ( + registry.shader_compute("conv2d_im2col_texture3d_half"), compute_count); + EXPECT_EQ( + registry.shader_memory("conv2d_im2col_buffer_float"), negative_count); + EXPECT_EQ( + registry.shader_compute("conv2d_im2col_buffer_float"), negative_count); + } +} + +TEST( + BenchmarkThroughputRegistry, + PrefixRegistrationsRejectInvalidAndDuplicateNames) { + ThroughputRegistry registry; + registry.register_shader_memory_prefix("conv2d", memory_count); + registry.register_shader_compute_prefix("conv2d", memory_count); + EXPECT_THROW( + registry.register_shader_memory_prefix("conv2d", compute_count), + std::invalid_argument); + EXPECT_THROW( + registry.register_shader_compute_prefix("conv2d", compute_count), + std::invalid_argument); + EXPECT_EQ(registry.shader_memory("conv2d_half"), memory_count); + EXPECT_EQ(registry.shader_compute("conv2d_half"), memory_count); + EXPECT_THROW( + registry.register_shader_memory_prefix("", memory_count), + std::invalid_argument); + EXPECT_THROW( + registry.register_shader_memory_prefix("conv2d", nullptr), + std::invalid_argument); + EXPECT_THROW( + registry.register_shader_compute_prefix("", memory_count), + std::invalid_argument); + EXPECT_THROW( + registry.register_shader_compute_prefix("conv2d", nullptr), + std::invalid_argument); +} + +TEST(BenchmarkThroughputRegistry, NullShaderCountersAreRejectedBeforeLookup) { + ThroughputRegistry registry; + registry.register_shader_memory_prefix("conv2d", memory_count); + registry.register_shader_compute_prefix("conv2d", compute_count); + EXPECT_THROW( + registry.register_shader_memory_prefix("conv2d_gemm", nullptr), + std::invalid_argument); + EXPECT_THROW( + registry.register_shader_compute_prefix("conv2d_gemm", nullptr), + std::invalid_argument); + EXPECT_THROW( + registry.register_shader_memory("conv2d_gemm_half", nullptr), + std::invalid_argument); + EXPECT_THROW( + registry.register_shader_compute("conv2d_gemm_half", nullptr), + std::invalid_argument); + EXPECT_EQ(registry.shader_memory("conv2d_gemm_half"), memory_count); + EXPECT_EQ(registry.shader_compute("conv2d_gemm_half"), compute_count); +} + +TEST( + BenchmarkThroughputRegistry, + StaticPrefixCountersReceiveResolvedShaderMetadata) { + const auto work = + estimate_work({{"shader_name", "test_prefix_float"}}, true, true); + EXPECT_EQ(work.memory_bytes, 40); + EXPECT_EQ(work.compute_flops, 20); + EXPECT_TRUE(work.diagnostics.empty()); +} + +TEST( + BenchmarkThroughputRegistry, + FamilyPrefixesCoverUnenumeratedShaderVariants) { + const auto& registry = throughput_registry(); + for (const char* name : + {"conv2d_gemm_texture2d_half", + "conv2d_new_variant_float", + "q8ta_conv2d_float", + "q8ta_im2col_new_variant", + "linear_vec_bias_tile_row_4_buffer_texture2d_half", + "buffer_to_buffer_new_dtype"}) { + EXPECT_EQ(registry.shader_memory(name), throughput::tensor_traffic); + } + EXPECT_EQ( + registry.shader_compute( + "linear_vec_bias_tile_row_4_buffer_texture2d_half"), + registry.shader_compute( + "linear_vec_bias_tile_row_1_buffer_texture2d_half")); + EXPECT_EQ( + registry.shader_compute( + "linear_vec_tile_row_4_texture3d_texture2d_float"), + registry.shader_compute( + "linear_vec_tile_row_1_texture3d_texture2d_float")); + EXPECT_NE( + registry.shader_compute("linear_vec_bias_float"), + registry.shader_compute("linear_vec_float")); + EXPECT_EQ( + registry.shader_memory("q4gsw_linear_gemm__w_8x8_nc_buffer_half"), + registry.shader_memory("q4gsw_linear_gemm__w_4x8_nc_buffer_float")); + EXPECT_EQ( + registry.shader_memory("q4gsw_linear_gemm__tin__w_8x8_nc_buffer_half"), + registry.shader_memory("q4gsw_linear_gemm__tin__w_4x8_nc_buffer_half")); + EXPECT_NE( + registry.shader_memory("q4gsw_linear_gemm__tin__float"), + registry.shader_memory("q4gsw_linear_gemm__float")); + EXPECT_EQ( + registry.shader_compute("binary_add_buffer_float"), + registry.shader_compute("binary_add_texture3d_float")); +} + +TEST( + BenchmarkThroughputRegistry, + CounterFunctionsCanBeSharedAcrossRegistrations) { + ThroughputRegistry registry; + registry.register_operator_memory("first", memory_count); + registry.register_operator_memory("second", memory_count); + registry.register_shader_memory("first", memory_count); + registry.register_shader_compute_prefix("second", memory_count); + EXPECT_EQ(registry.operator_memory("first"), memory_count); + EXPECT_EQ(registry.operator_memory("second"), memory_count); + EXPECT_EQ(registry.shader_memory("first"), memory_count); + EXPECT_EQ(registry.shader_compute("second_half"), memory_count); +} + +TEST(BenchmarkThroughputRegistry, DuplicateRegistrationsKeepOriginalMapping) { + ThroughputRegistry registry; + registry.register_operator_memory("same", memory_count); + registry.register_operator_compute("same", memory_count); + registry.register_shader_memory("same", memory_count); + registry.register_shader_compute("same", memory_count); + EXPECT_THROW( + registry.register_operator_memory("same", compute_count), + std::invalid_argument); + EXPECT_THROW( + registry.register_operator_compute("same", compute_count), + std::invalid_argument); + EXPECT_THROW( + registry.register_shader_memory("same", compute_count), + std::invalid_argument); + EXPECT_THROW( + registry.register_shader_compute("same", compute_count), + std::invalid_argument); + EXPECT_EQ(registry.operator_memory("same"), memory_count); + EXPECT_EQ(registry.operator_compute("same"), memory_count); + EXPECT_EQ(registry.shader_memory("same"), memory_count); + EXPECT_EQ(registry.shader_compute("same"), memory_count); +} + +TEST(BenchmarkThroughputRegistry, InvalidNamesAndFunctionsAreRejected) { + ThroughputRegistry registry; + EXPECT_THROW( + registry.register_operator_memory("", memory_count), + std::invalid_argument); + EXPECT_THROW( + registry.register_operator_compute("", compute_count), + std::invalid_argument); + EXPECT_THROW( + registry.register_shader_memory("", memory_count), std::invalid_argument); + EXPECT_THROW( + registry.register_shader_compute("", compute_count), + std::invalid_argument); + EXPECT_THROW( + registry.register_operator_memory("same", nullptr), + std::invalid_argument); + EXPECT_THROW( + registry.register_operator_compute("same", nullptr), + std::invalid_argument); + EXPECT_THROW(registry.shader_compute("missing"), std::invalid_argument); + EXPECT_THROW(registry.operator_memory("missing"), std::invalid_argument); +} + +TEST( + BenchmarkThroughputRegistry, + StaticRegistrationsSelectOperatorAndShaderCounters) { + const auto op = + estimate_work({{"operator_name", "test_counter_case"}}, true, true); + const auto shader = + estimate_work({{"shader_name", "test_counter_case"}}, true, true); + EXPECT_EQ(op.memory_bytes, 40); + EXPECT_EQ(op.compute_flops, 20); + EXPECT_EQ(shader.memory_bytes, 20); + EXPECT_EQ(shader.compute_flops, 40); + EXPECT_TRUE(op.diagnostics.empty()); +} + +TEST(BenchmarkThroughputRegistry, MissingCountersAreOptionalWithDiagnostics) { + const auto absent = estimate_work({{"shader_name", "missing"}}, true, true); + EXPECT_FALSE(absent.memory_bytes); + EXPECT_FALSE(absent.compute_flops); + EXPECT_EQ(absent.diagnostics.size(), 2); + const auto disabled = estimate_work(Json(), false, false); + EXPECT_FALSE(disabled.memory_bytes); + EXPECT_TRUE(disabled.diagnostics.empty()); +} + +TEST(BenchmarkThroughputRegistry, InvalidWorkCountsDoNotHideValidOtherCounter) { + for (const char* name : + {"test_negative", "test_infinite", "test_nan", "test_throwing"}) { + const auto result = estimate_work( + {{"shader_name", name}}, true, name == std::string("test_throwing")); + EXPECT_FALSE(result.memory_bytes) << name; + ASSERT_FALSE(result.diagnostics.empty()) << name; + if (name == std::string("test_throwing")) { + EXPECT_EQ(result.compute_flops, 20); + EXPECT_NE( + result.diagnostics[0].find("bad work shape"), std::string::npos); + } + } +} + +TEST( + BenchmarkThroughputRegistry, + LinearCountsLogicalTrafficAndMacsIncludingBias) { + const auto work = estimate_work(linear_case(), true, true); + EXPECT_EQ(work.memory_bytes, 120); + EXPECT_EQ(work.compute_flops, 56); + EXPECT_TRUE(work.diagnostics.empty()); + Json half = linear_case(); + for (auto& arg : half["args"]) { + arg["tensor"]["dtype"] = "float16"; + } + EXPECT_EQ(estimate_work(half, true, false).memory_bytes, 60); + Json no_bias = linear_case(); + no_bias["args"][2] = nullptr; + const auto without_bias = estimate_work(no_bias, true, true); + EXPECT_EQ(without_bias.memory_bytes, 104); + EXPECT_EQ(without_bias.compute_flops, 48); +} + +TEST( + BenchmarkThroughputRegistry, + Q4CountsPackedBytesAndArithmeticEquivalentMacs) { + const Json spec = { + {"operator_name", "et_vk.q4gsw_linear.default"}, + {"args", + {{{"tensor", tensor({2, 32})}}, + {{"tensor", tensor({4, 16}, "uint8", "read", "constant")}}, + {{"tensor", tensor({1, 4}, "float32", "read", "constant")}}, + {{"int", 32}}, + {{"tensor", tensor({4}, "float32", "read", "constant")}}, + {{"tensor", tensor({2, 4}, "float32", "write")}}}}}; + const auto work = estimate_work(spec, true, true); + EXPECT_EQ(work.memory_bytes, 384); + EXPECT_EQ(work.compute_flops, 520); +} + +TEST( + BenchmarkThroughputRegistry, + ConvolutionCountsGroupedStrideAndPaddingCase) { + const Json spec = { + {"operator_name", "aten.convolution.default"}, + {"args", + {{{"tensor", tensor({1, 4, 5, 5})}}, + {{"tensor", tensor({8, 2, 3, 3}, "float32", "read", "constant")}}, + {{"tensor", tensor({8}, "float32", "read", "constant")}}, + {{"int_list", {2, 2}}}, + {{"int_list", {1, 1}}}, + {{"int_list", {1, 1}}}, + {{"bool", false}}, + {{"int_list", {0, 0}}}, + {{"int", 2}}, + {{"tensor", tensor({1, 8, 3, 3}, "float32", "write")}}}}}; + const auto work = estimate_work(spec, true, true); + EXPECT_EQ(work.memory_bytes, 1296); + EXPECT_EQ(work.compute_flops, 2664); + Json transposed = spec; + transposed["args"][6]["bool"] = true; + EXPECT_FALSE(estimate_work(transposed, false, true).compute_flops); +} + +TEST( + BenchmarkThroughputRegistry, + Q8CountsPackedInt8ElementsAndCorrectionScales) { + const Json spec = { + {"operator_name", "et_vk.q8ta_conv2d.default"}, + {"args", + {{{"tensor", tensor({1, 4, 5, 5}, "int8x4")}}, + {{"float", 0.25}}, + {{"int", 0}}, + {{"tensor", tensor({8, 36}, "int8", "read", "constant")}}, + {{"tensor", tensor({8}, "int32", "read", "constant")}}, + {{"tensor", tensor({8}, "float32", "read", "constant")}}, + {{"float", 0.25}}, + {{"int", 0}}, + {{"tensor", tensor({8}, "float32", "read", "constant")}}, + {{"int_list", {3, 3}}}, + {{"int_list", {1, 1}}}, + {{"int_list", {1, 1}}}, + {{"int_list", {1, 1}}}, + {{"int", 1}}, + {{"string", "none"}}, + {{"tensor", tensor({1, 8, 5, 5}, "int8x4", "write")}}}}}; + const auto work = estimate_work(spec, true, true); + EXPECT_EQ(work.memory_bytes, 684); + EXPECT_EQ(work.compute_flops, 14600); +} + +TEST( + BenchmarkThroughputRegistry, + CopyAndBroadcastAdditionHaveLogicalWorkCounts) { + const Json copy = { + {"shader_name", "buffer_to_buffer_float"}, + {"bindings", + {{"input", tensor({8})}, + {"output", tensor({8}, "float32", "write")}, + {"meta", {{"type", "meta_ubo"}}}}}}; + const auto copied = estimate_work(copy, true, true); + EXPECT_EQ(copied.memory_bytes, 64); + EXPECT_EQ(copied.compute_flops, 0); + const Json add = { + {"shader_name", "binary_add_texture3d_float"}, + {"bindings", + {{"a", tensor({2, 1}, "float32", "read", "texture3d")}, + {"b", tensor({3}, "float32", "read", "texture3d")}, + {"out", tensor({2, 3}, "float32", "write", "texture3d")}}}}; + const auto added = estimate_work(add, true, true); + EXPECT_EQ(added.memory_bytes, 44); + EXPECT_EQ(added.compute_flops, 12); +} + +TEST( + BenchmarkThroughputRegistry, + ShaderLinearCountsActualBindingsWithoutParentWork) { + const Json spec = { + {"shader_name", "linear_vec_bias_tile_row_1_buffer_texture2d_float"}, + {"bindings", + {{"binding_0", tensor({1, 8}, "float32", "write")}, + {"binding_1", tensor({1, 4})}, + {"binding_2", tensor({8, 4}, "float32", "read", "texture2d")}, + {"binding_3", tensor({8})}}}}; + const auto work = estimate_work(spec, true, true); + EXPECT_EQ(work.memory_bytes, 208); + EXPECT_EQ(work.compute_flops, 72); + EXPECT_TRUE(work.diagnostics.empty()); + Json missing = spec; + missing["shader_name"] = "unregistered_child_shader"; + EXPECT_FALSE(estimate_work(missing, true, true).memory_bytes); +} + +TEST( + BenchmarkThroughputRegistry, + LogicalTrafficCountsReadWriteTwiceAndDeduplicatesRefs) { + const Json spec = { + {"shader_name", "buffer_to_buffer_float"}, + {"bindings", + {{"x", + {{"name", "x"}, + {"type", "buffer"}, + {"sizes", {8}}, + {"dtype", "float32"}, + {"access", "read_write"}}}, + {"x_alias", + {{"ref", "x"}, + {"type", "buffer"}, + {"sizes", {8}}, + {"dtype", "float32"}, + {"access", "read"}}}}}}; + EXPECT_EQ(estimate_work(spec, true, false).memory_bytes, 64); + Json malformed = spec; + malformed["bindings"]["x"]["dtype"] = "unknown"; + const auto result = estimate_work(malformed, true, false); + EXPECT_FALSE(result.memory_bytes); + EXPECT_FALSE(result.diagnostics.empty()); +} + +TEST(BenchmarkThroughputRegistry, Q4ShaderMemoryExcludesInactiveBindingSlots) { + Json fp32 = { + {"shader_name", "q4gsw_linear_gemm__w_4x8_nc_buffer_float"}, + {"bindings", + {{"binding_0", tensor({4, 8}, "float32", "write")}, + {"binding_1", tensor({4, 32})}, + {"binding_2", tensor(Json::array())}, + {"binding_3", tensor({8, 4}, "int32")}, + {"binding_4", tensor({1, 8})}, + {"binding_5", tensor({8})}}}}; + for (size_t i = 0; i < 6; ++i) { + fp32["bindings"]["binding_" + std::to_string(i)]["binding_index"] = i; + } + const auto fp32_work = estimate_work(fp32, true, true); + EXPECT_EQ(fp32_work.memory_bytes, 832); + EXPECT_FALSE(fp32_work.compute_flops); + Json fp16 = fp32; + fp16["shader_name"] = "q4gsw_linear_gemm__tin__w_4x8_nc_buffer_half"; + fp16["bindings"]["binding_2"]["sizes"] = {32, 4}; + for (size_t i : {0, 1, 2, 4, 5}) { + fp16["bindings"]["binding_" + std::to_string(i)]["dtype"] = "float16"; + } + const auto fp16_work = estimate_work(fp16, true, false); + EXPECT_EQ(fp16_work.memory_bytes, 480); + EXPECT_TRUE(fp16_work.diagnostics.empty()); + fp16["bindings"]["binding_5"]["sizes"] = Json::array(); + EXPECT_EQ(estimate_work(fp16, true, false).memory_bytes, 464); +} + +} // namespace +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/test/throughput_report_test.cpp b/backends/vulkan/test/ops/benchmark_v2/framework/test/throughput_report_test.cpp new file mode 100644 index 00000000000..b5493e54514 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/test/throughput_report_test.cpp @@ -0,0 +1,135 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include + +#include +#include + +namespace vkcompute::shader_profiling { +namespace { + +const Json kBoth = { + {"show_memory_throughput", true}, + {"show_compute_throughput", true}}; + +TEST( + BenchmarkThroughputReport, + CalculatesRatesPerSampleWithCorrectBinaryAndFlopUnits) { + const WorkAmounts work{1073741824.0, 2000000000.0, {}}; + const auto report = + throughput_report(kBoth, work, 2, {2000000, 4000000}, {4000000, 8000000}); + const auto& gpu = report.at("gpu_timestamp"); + const auto& memory = gpu.at("memory_gib_per_s"); + EXPECT_DOUBLE_EQ(memory.at("min"), 0.5); + EXPECT_DOUBLE_EQ(memory.at("max"), 1.0); + EXPECT_DOUBLE_EQ(memory.at("mean"), 0.75); + EXPECT_NEAR(memory.at("p90").get(), 0.95, 1e-12); + EXPECT_DOUBLE_EQ(gpu.at("memory_mib_per_s").at("mean"), 768); + EXPECT_DOUBLE_EQ(gpu.at("compute_gflops").at("mean"), 1.5); + EXPECT_EQ(report.at("gpu_timestamp_batch"), gpu); + EXPECT_EQ( + report.at("wall_time").at("memory_gib_per_s"), + Report::array({0.5, 0.25})); + EXPECT_EQ( + report.at("wall_time").at("compute_gflops"), Report::array({1.0, 0.5})); + EXPECT_EQ(report.at("wall_time_batch"), report.at("wall_time")); +} + +TEST(BenchmarkThroughputReport, FlagsControlOnlyRequestedMetrics) { + const WorkAmounts work{1024.0, 4.0, {}}; + EXPECT_TRUE(throughput_report({}, work, 1, {1}, {2}).empty()); + const auto memory = + throughput_report({{"show_memory_throughput", true}}, work, 1, {1}, {2}); + EXPECT_TRUE(memory.contains("memory_bytes")); + EXPECT_FALSE(memory.contains("compute_flops")); + EXPECT_FALSE(memory.at("gpu_timestamp").contains("compute_gflops")); + const auto compute = + throughput_report({{"show_compute_throughput", true}}, work, 1, {1}, {2}); + EXPECT_FALSE(compute.contains("memory_bytes")); + EXPECT_TRUE(compute.contains("compute_flops")); + EXPECT_FALSE(compute.at("gpu_timestamp").contains("memory_gib_per_s")); +} + +TEST(BenchmarkThroughputReport, UnknownCountsAreNullAndDoNotEraseValidMetrics) { + const WorkAmounts work{std::nullopt, 4.0, {"No memory counter"}}; + const auto report = throughput_report(kBoth, work, 1, {2}, {4}); + EXPECT_TRUE(report.at("memory_bytes").is_null()); + EXPECT_TRUE(report.at("gpu_timestamp").at("memory_gib_per_s").is_null()); + EXPECT_DOUBLE_EQ( + report.at("gpu_timestamp").at("compute_gflops").at("mean"), 0.002); + EXPECT_EQ(report.at("diagnostics"), Report::array({"No memory counter"})); +} + +TEST( + BenchmarkThroughputReport, + ZeroWorkAndUnavailableTimestampsRemainDistinct) { + const WorkAmounts work{0.0, 0.0, {}}; + const auto report = throughput_report(kBoth, work, 1, {}, {1, 2}); + EXPECT_TRUE(report.at("gpu_timestamp").at("compute_gflops").is_null()); + EXPECT_EQ(report.at("wall_time").at("compute_gflops"), Report::array({0, 0})); + EXPECT_DOUBLE_EQ( + throughput_report(kBoth, work, 1, {1}, {}) + .at("gpu_timestamp") + .at("compute_gflops") + .at("mean"), + 0); +} + +TEST(BenchmarkThroughputReport, InvalidIntervalsCannotGenerateInfiniteRates) { + const WorkAmounts work{4.0, 4.0, {}}; + const auto report = throughput_report(kBoth, work, 1, {0, 1}, {0, 2}); + EXPECT_TRUE(report.at("gpu_timestamp").at("compute_gflops").is_null()); + EXPECT_TRUE(report.at("wall_time").at("compute_gflops")[0].is_null()); + EXPECT_DOUBLE_EQ(report.at("wall_time").at("compute_gflops")[1], 0.002); + EXPECT_THROW( + throughput_report(kBoth, work, 0, {1}, {2}), std::invalid_argument); +} + +TEST( + BenchmarkThroughputReport, + ShaderBreakdownUsesWorkAndSamplesForEachDispatch) { + const Report dispatches = + Report::array({{{"shader_name", "first"}}, {{"shader_name", "second"}}}); + const std::vector work = { + {1048576.0, 1000000000.0, {}}, {2097152.0, 3000000000.0, {}}}; + const auto result = shader_breakdown_report( + dispatches, {{1000000, 2000000, 1000000, 2000000}}, 2, kBoth, work); + EXPECT_DOUBLE_EQ( + result[0] + .at("throughput") + .at("gpu_timestamp") + .at("memory_mib_per_s") + .at("mean"), + 1); + EXPECT_DOUBLE_EQ( + result[1] + .at("throughput") + .at("gpu_timestamp") + .at("memory_mib_per_s") + .at("mean"), + 1); + EXPECT_DOUBLE_EQ( + result[0] + .at("throughput") + .at("gpu_timestamp") + .at("compute_gflops") + .at("mean"), + 1); + EXPECT_DOUBLE_EQ( + result[1] + .at("throughput") + .at("gpu_timestamp") + .at("compute_gflops") + .at("mean"), + 1.5); + EXPECT_FALSE(result[0].at("throughput").contains("wall_time")); +} + +} // namespace +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/framework/test/validation_test.cpp b/backends/vulkan/test/ops/benchmark_v2/framework/test/validation_test.cpp new file mode 100644 index 00000000000..43f5d064ec5 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/framework/test/validation_test.cpp @@ -0,0 +1,254 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include +#include +#include + +#include + +#include +#include + +namespace vkcompute::shader_profiling { +using BenchmarkValidation = VulkanTest; + +namespace { + +Json copy_case(const Json& gwg) { + return parse_configs({{"shader", "buffer_to_buffer_float"}, + {"gwg_size", gwg}, + {"lwg_size", {1, 1, 1}}, + {"bindings", + Json::array( + {{{"buffer", + {{"name", "out"}, + {"sizes", {8}}, + {"access", "write"}, + {"data", {{"fill", 0}}}}}}, + {{"buffer", + {{"name", "in"}, + {"sizes", {8}}, + {"data", {1, 2, 3, 4, 5, 6, 7, 8}}}}}, + {{"meta_ubo", "out"}, {"field", "numel"}}})}}) + .at(0); +} + +TEST_F(BenchmarkValidation, StagedShaderCopyPassesWithoutBenchmarking) { + const Json spec = copy_case({8, 1, 1}); + const Report result = benchmark_shader(spec, prepare_tensor_data(spec), true); + EXPECT_EQ(result.at("validation").at("status"), "PASS"); + EXPECT_FALSE(result.contains("gpu_timestamp_shader_us")); + EXPECT_FALSE(result.contains("wall_time_shader_us")); +} + +TEST_F(BenchmarkValidation, GraphCreationReusesPreparedHostData) { + const Json spec = copy_case({8, 1, 1}); + auto host_data = prepare_tensor_data(spec); + host_data.at("in") = + make_data({{"data", {-1, -2, -3, -4, -5, -6, -7, -8}}}, 8); + const auto expected = host_data; + const auto* input_data = host_data.at("in").data(); + { + ComputeGraph graph{GraphConfig{}}; + create_tensors(spec, graph, host_data, true); + EXPECT_EQ(host_data, expected); + EXPECT_EQ(host_data.at("in").data(), input_data); + } + { + ComputeGraph graph{GraphConfig{}}; + create_tensors(spec, graph, host_data, false); + EXPECT_EQ(host_data, expected); + EXPECT_EQ(host_data.at("in").data(), input_data); + EXPECT_TRUE(graph.inputs().empty()); + EXPECT_TRUE(graph.execute_nodes().empty()); + } +} + +TEST_F(BenchmarkValidation, ValidationSelectionRequiresWritableTensors) { + Json spec = copy_case({8, 1, 1}); + spec["validation_method"] = {{"function", "is_close"}, {"outputs", {"out"}}}; + ComputeGraph graph{GraphConfig{}}; + const auto host_data = prepare_tensor_data(spec); + const auto tensors = create_tensors(spec, graph, host_data); + const Json accesses{{"out", "write"}, {"in", "read"}}; + const auto outputs = add_validation_outputs(spec, graph, tensors, accesses); + ASSERT_EQ(outputs.size(), 1); + EXPECT_EQ(outputs.begin()->first, "out"); + spec["validation_method"]["outputs"] = {"in"}; + EXPECT_THROW( + add_validation_outputs(spec, graph, tensors, accesses), + std::invalid_argument); +} + +TEST_F(BenchmarkValidation, RejectsWrongOutputShapeWithMatchingElementCount) { + Json spec = copy_case({8, 1, 1}); + spec["tensors"]["in"]["sizes"] = {1, 8}; + spec["tensors"]["out"]["sizes"] = {1, 8}; + ComputeGraph graph{GraphConfig{}}; + const auto host_data = prepare_tensor_data(spec); + const auto tensors = create_tensors(spec, graph, host_data); + graph.virtual_resize(tensors.at("out"), {2, 4}); + const ValueRef staging = + graph.add_staging(vkapi::kFloat, 8, vkapi::CopyDirection::DEVICE_TO_HOST); + graph.prepare(); + const std::array values = {1, 2, 3, 4, 5, 6, 7, 8}; + graph.maybe_cast_and_copy_into_staging( + staging, values.data(), values.size(), vkapi::kFloat); + EXPECT_THROW( + validate_outputs(spec, graph, tensors, {{"out", staging}}, host_data), + std::invalid_argument); +} + +TEST_F(BenchmarkValidation, IncompleteShaderGridReportsOutputMismatches) { + const Json spec = copy_case({1, 1, 1}); + const Report result = benchmark_shader(spec, prepare_tensor_data(spec), true); + EXPECT_EQ(result.at("validation").at("status"), "FAIL"); + EXPECT_EQ( + result.at("validation").at("outputs").at("out").at("mismatch_count"), 7); +} + +TEST_F(BenchmarkValidation, StagedTextureAdditionUsesOneTargetDescriptorSet) { + const Json spec = + parse_configs( + {{"shader", "binary_add_texture3d_float"}, + {"gwg_size", "default_pick_gwg"}, + {"lwg_size", {10, 10, 1}}, + {"bindings", + Json::array( + {{{"texture3d", + {{"name", "out"}, + {"sizes", {1, 1, 1, 8}}, + {"access", "write"}, + {"data", {{"fill", 0}}}}}}, + {{"texture3d", + {{"name", "a"}, + {"sizes", {1, 1, 1, 8}}, + {"data", {1, 2, 3, 4, 5, 6, 7, 8}}}}}, + {{"texture3d", + {{"name", "b"}, + {"sizes", {1, 1, 1, 8}}, + {"data", {-1, -2, -3, -4, -5, -6, -7, -8}}}}}, + {{"meta_ubo", "out"}}, + {{"meta_ubo", "a"}}, + {{"meta_ubo", "b"}}})}, + {"specialization_constants", + {{{"layout", "out"}}, {{"layout", "a"}}, {{"layout", "b"}}}}, + {"push_constants", {{{"dtype", "float32"}, {"data", {2}}}}}}) + .at(0); + EXPECT_EQ( + benchmark_shader(spec, prepare_tensor_data(spec), true) + .at("validation") + .at("status"), + "PASS"); +} + +TEST_F(BenchmarkValidation, StagedLinearUsesConstantWeightsAndOptionalBias) { + const Json spec = + parse_configs( + {{"operator", "aten.linear.default"}, + {"args_sweep", + Json::array({Json::array( + {{{"tensor", + {{"name", "x"}, {"sizes", {1, 4}}, {"data", {1, 2, 3, 4}}}}}, + {{"tensor", + {{"name", "w"}, + {"sizes", {4, 4}}, + {"storage", "constant"}, + {"data", + {1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1}}}}}, + nullptr, + {{"tensor", + {{"name", "y"}, + {"sizes", {1, 4}}, + {"data", {{"fill", 0}}}}}}})})}}) + .at(0); + for (const char* dtype : {"float32", "float16"}) { + Json typed = spec; + for (auto& tensor : typed["tensors"].items()) { + tensor.value()["dtype"] = dtype; + } + typed.update( + {{"target_ms", 0.01}, + {"benchmark_runs", 2}, + {"warmup_runs", 0}, + {"max_dispatches", 2}}); + const auto host_data = prepare_tensor_data(typed); + const auto expected = host_data; + EXPECT_EQ( + benchmark_operator(typed, host_data, true) + .at("validation") + .at("status"), + "PASS"); + EXPECT_TRUE( + benchmark_operator(typed, host_data).contains("wall_time_operator_us")); + EXPECT_EQ( + benchmark_operator(typed, host_data, true) + .at("validation") + .at("status"), + "PASS"); + EXPECT_EQ(host_data, expected); + EXPECT_EQ( + host_data.at("w").size(), std::string(dtype) == "float16" ? 32 : 64); + } +} + +TEST_F(BenchmarkValidation, BenchmarkAddsRegisteredWorkAndWallThroughput) { + Json spec = copy_case({8, 1, 1}); + spec.update( + {{"show_memory_throughput", true}, + {"show_compute_throughput", true}, + {"target_ms", 0.01}, + {"benchmark_runs", 2}, + {"warmup_runs", 0}, + {"max_dispatches", 2}}); + const Report result = benchmark_shader(spec, prepare_tensor_data(spec)); + const auto& throughput = result.at("throughput"); + EXPECT_DOUBLE_EQ(throughput.at("memory_bytes"), 64); + EXPECT_DOUBLE_EQ(throughput.at("compute_flops"), 0); + EXPECT_EQ(throughput.at("wall_time").at("memory_gib_per_s").size(), 2); + EXPECT_EQ( + throughput.at("wall_time").at("compute_gflops"), Report::array({0, 0})); + const double count = result.at("dispatches_per_batch"); + for (size_t i = 0; i < 2; ++i) { + const double expected = 64.0 * count * 1e6 / + result.at("wall_time_batch_us")[i].get() / 1073741824.0; + EXPECT_NEAR( + throughput.at("wall_time").at("memory_gib_per_s")[i].get(), + expected, + 1e-12); + } +} + +TEST_F(BenchmarkValidation, UnsupportedShaderIsRejectedBeforePipelineCreation) { + ComputeGraph graph{GraphConfig{}}; + if (graph.context()->adapter_ptr()->supports_int8_dot_product()) { + GTEST_SKIP() << "Requires a device without integer dot-product support"; + } + const auto shader = VK_KERNEL_FROM_STR("q8ta_conv2d_pw_float"); + ASSERT_TRUE(shader.requires_integer_dot_product); + EXPECT_THROW( + graph.register_pipeline_to_create( + shader, LocalWorkGroup(1, 1, 1), {}, {}), + vkapi::ShaderNotSupportedError); +} + +TEST_F( + BenchmarkValidation, + UnrecognizedSpirvCapabilityIsRejectedBeforePipelineCreation) { + ComputeGraph graph{GraphConfig{}}; + vkapi::ShaderInfo shader; + shader.kernel_name = "unsupported_capability"; + shader.required_capabilities = {1, 65535}; + EXPECT_THROW( + graph.register_pipeline_to_create( + shader, LocalWorkGroup(1, 1, 1), {}, {}), + vkapi::ShaderNotSupportedError); +} + +} // namespace +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/MigrationReferenceTest.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/MigrationReferenceTest.cpp new file mode 100644 index 00000000000..e20a5a00081 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/MigrationReferenceTest.cpp @@ -0,0 +1,221 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace vkcompute::shader_profiling { +namespace { +Json tensor(const char* name) { + return {{"tensor", name}}; +} +TEST(MigratedReferences, Conv1dDepthwiseBatchStridePadding) { + const Json spec = { + {"args", + {tensor("input"), + tensor("weight"), + nullptr, + {{"int_list", {2}}}, + {{"int_list", {1}}}, + {{"int_list", {1}}}, + {{"bool", false}}, + {{"int_list", {0}}}, + {{"int", 2}}, + tensor("output")}}}; + const reference::TensorMap tensors = { + {"input", + {{2, 2, 4}, {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}}}, + {"weight", {{2, 1, 3}, {1, 1, 1, 2, 2, 2}}}}; + const auto out = reference::conv1d_reference(spec, tensors, 2); + EXPECT_EQ(out.sizes, (std::vector{2, 2, 2})); + EXPECT_EQ(out.values, (std::vector{3, 9, 22, 42, 19, 33, 54, 90})); +} +TEST(MigratedReferences, BatchedRectangularMatmul) { + const Json spec = { + {"operator", "aten.bmm.default"}, + {"args", {tensor("a"), tensor("b"), tensor("output")}}}; + const reference::TensorMap tensors = { + {"a", {{2, 1, 2}, {1, 2, 3, 4}}}, + {"b", {{2, 2, 3}, {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12}}}}; + const auto out = reference::matmul_reference(spec, tensors, 2); + EXPECT_EQ(out.sizes, (std::vector{2, 1, 3})); + EXPECT_EQ(out.values, (std::vector{9, 12, 15, 61, 68, 75})); +} +TEST(MigratedReferences, AddmmAlphaBeta) { + const Json spec = { + {"operator", "aten.addmm.default"}, + {"args", + {tensor("bias"), + tensor("a"), + tensor("b"), + {{"float", 3}}, + {{"float", 2}}, + tensor("output")}}}; + const reference::TensorMap tensors = { + {"a", {{1, 2}, {1, 2}}}, + {"b", {{2, 2}, {3, 4, 5, 6}}}, + {"bias", {{2}, {10, 20}}}}; + EXPECT_EQ( + reference::matmul_reference(spec, tensors, 1).values, + (std::vector{56, 92})); +} +TEST(MigratedReferences, EmbeddingNibbleOrderAndGroupedScales) { + Json spec = { + {"args", + {tensor("w"), + tensor("s"), + {{"int", 2}}, + tensor("i"), + {{"bool", false}}, + tensor("output")}}}; + const reference::TensorMap tensors = { + {"w", {{2, 2}, {0x9a, 0xbc, 0xde, 0xf8}}}, + {"s", {{2, 2}, {1, 2, 3, 4}}}, + {"i", {{2}, {1, 0}}}}; + EXPECT_EQ( + reference::embedding_q4gsw_reference(spec, tensors, 1).values, + (std::vector{15, 18, 28, 0, 1, 2, 6, 8})); + spec["args"][4]["bool"] = true; + EXPECT_EQ( + reference::embedding_q4gsw_reference(spec, tensors, 1).values, + (std::vector{18, 15, 0, 28, 2, 1, 8, 6})); +} +TEST(MigratedReferences, DynamicQ4Clipping) { + Json spec = { + {"args", + {tensor("x"), + tensor("is"), + tensor("zp"), + tensor("w"), + tensor("sum"), + tensor("s"), + {{"int", 2}}, + nullptr, + tensor("output")}}, + {"route", {{"name", "auto"}}}}; + const reference::TensorMap tensors = { + {"x", {{1, 2}, {100, -100}}}, + {"is", {{1, 1}, {0.5}}}, + {"zp", {{1, 1}, {0}}}, + {"w", {{1, 1}, {0x99}}}, + {"s", {{1, 1}, {1}}}}; + EXPECT_EQ( + reference::dq8ca_q4gsw_reference(spec, tensors, 1).values, + (std::vector{-0.5})); +} + +TEST(MigratedReferences, DynamicQ4UsesWeightOnlyFallbackWithoutIntegerDot) { + Json spec = { + {"args", + {tensor("x"), + tensor("is"), + tensor("zp"), + tensor("w"), + tensor("sum"), + tensor("s"), + {{"int", 2}}, + nullptr, + tensor("output")}}, + {"reference_context", {{"int8_dot_product", false}}}}; + const reference::TensorMap tensors = { + {"x", {{1, 2}, {100, -100}}}, + {"w", {{1, 1}, {0x99}}}, + {"s", {{1, 1}, {1}}}}; + EXPECT_EQ( + reference::dq8ca_q4gsw_reference(spec, tensors, 1).values, + (std::vector{0})); +} + +TEST(MigratedReferences, DynamicQ4RejectsMalformedReferenceCapabilities) { + Json spec = { + {"args", + {tensor("x"), + tensor("is"), + tensor("zp"), + tensor("w"), + tensor("sum"), + tensor("s"), + {{"int", 2}}, + nullptr, + tensor("output")}}, + {"reference_context", {{"int8_dot_product", "false"}}}}; + EXPECT_THROW( + reference::dq8ca_q4gsw_reference(spec, reference::TensorMap{}, 1), + std::invalid_argument); +} +TEST(MigratedReferences, SdpaFusedCacheUpdateAndCausalPrefix) { + Json spec = { + {"operator", "sdpa_with_kv_cache.default"}, + {"args", + {tensor("q"), + tensor("pk"), + tensor("pv"), + tensor("k"), + tensor("v"), + {{"int", 1}}, + {{"int", 2}}, + nullptr, + {{"float", 0}}, + {{"bool", true}}, + {{"float", 1}}, + tensor("output")}}}; + const reference::TensorMap tensors = { + {"q", {{1, 2, 2, 1}, {0, 0, 0, 0}}}, + {"k", {{1, 4, 1, 1}, {0, 0, 0, 0}}}, + {"v", {{1, 4, 1, 1}, {2, 100, 200, 1000}}}, + {"pk", {{1, 2, 1, 1}, {0, 0}}}, + {"pv", {{1, 2, 1, 1}, {4, 6}}}}; + const auto out = reference::sdpa_reference(spec, tensors, 2); + EXPECT_EQ(out.values, (std::vector{3, 3, 4, 4})); + spec["operator"] = "llama.custom_sdpa.default"; + spec["args"] = { + tensor("q"), + tensor("k"), + tensor("v"), + {{"int", 1}}, + nullptr, + {{"float", 0}}, + {{"bool", true}}, + {{"float", 1}}, + tensor("output")}; + const auto unchanged = reference::sdpa_reference(spec, tensors, 1); + EXPECT_FLOAT_EQ(unchanged.values[0], 51); + EXPECT_FLOAT_EQ(unchanged.values[2], 302.0f / 3); +} +TEST(MigratedReferences, NativeAddBroadcastAndAlpha) { + const Json spec = { + {"operator", "aten.add.Tensor"}, + {"args", {tensor("a"), tensor("b"), {{"float", 0.5}}, tensor("out")}}, + {"tensors", {{"out", {{"sizes", {2, 3}}}}}}}; + const reference::TensorMap tensors = { + {"a", {{2, 1}, {1, 2}}}, {"b", {{1, 3}, {2, 4, 6}}}}; + const auto out = reference::add_tensor_reference(spec, tensors, 2); + EXPECT_EQ(out.sizes, (std::vector{2, 3})); + EXPECT_EQ(out.values, (std::vector{2, 3, 4, 3, 4, 5})); +} +TEST(MigratedReferences, ChooseQparamsZeroAndAsymmetricRange) { + const Json spec = { + {"args", + {tensor("x"), + {{"int", -128}}, + {{"int", 127}}, + tensor("scale"), + tensor("zero_point")}}}; + const reference::TensorMap tensors = {{"x", {{2, 3}, {0, 0, 0, 0, 1, 2}}}}; + const auto out = reference::choose_qparams_reference(spec, tensors, 2); + EXPECT_FLOAT_EQ(out.at("scale")[0], 0.1f); + EXPECT_FLOAT_EQ(out.at("scale")[1], 2.0f / 255); + EXPECT_EQ(out.at("zero_point"), (std::vector{127, -128})); +} +} // namespace +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/attention/sdpa/SdpaReference.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/attention/sdpa/SdpaReference.cpp new file mode 100644 index 00000000000..2efbc15e818 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/attention/sdpa/SdpaReference.cpp @@ -0,0 +1,104 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include +#include +#include +#include +namespace vkcompute::shader_profiling::reference { +Tensor +sdpa_reference(const Json& spec, const TensorMap& tensors, size_t threads) { + const Tensor& q = tensor_arg(spec, 0, tensors); + const bool fused = spec.at("operator") == "sdpa_with_kv_cache.default"; + const Tensor& k = tensor_arg(spec, fused ? 3 : 1, tensors); + const Tensor& v = tensor_arg(spec, fused ? 4 : 2, tensors); + const int64_t pos = spec.at("args").at(fused ? 5 : 3).at("int"); + require( + q.sizes.size() == 4 && k.sizes.size() == 4 && v.sizes == k.sizes && + q.sizes[0] == 1, + "SDPA expects batch1 rank4 tensors"); + const size_t s = q.sizes[1], h = q.sizes[2], d = q.sizes[3], hk = k.sizes[2], + capacity = k.sizes[1]; + require( + h % hk == 0 && k.sizes[3] == int64_t(d) && pos >= 0 && + size_t(pos) + s <= capacity, + "Invalid SDPA head dimensions or cache position"); + const Json& scale_arg = spec.at("args").at(fused ? 10 : 7); + const float scale = scale_arg.is_null() ? 1 / std::sqrt(float(d)) + : scale_arg.at("float").get(); + const Tensor* pk = fused ? &tensor_arg(spec, 1, tensors) : nullptr; + const Tensor* pv = fused ? &tensor_arg(spec, 2, tensors) : nullptr; + Tensor out{q.sizes, std::vector(q.values.size())}; + parallel_for(s * h, threads, [&](size_t job) { + const size_t token = job / h, head = job % h, kvhead = head / (h / hk), + context = pos + token + 1; + std::vector probs(context); + float max = -std::numeric_limits::infinity(); + for (size_t c = 0; c < context; ++c) { + float dot = 0; + const Tensor& cache = pk && c >= size_t(pos) ? *pk : k; + const size_t ci = pk && c >= size_t(pos) ? c - pos : c; + for (size_t j = 0; j < d; ++j) + dot = std::fma( + float(q.values[job * d + j]), + float(cache.values[(ci * hk + kvhead) * d + j]), + dot); + probs[c] = dot * scale; + max = std::max(max, probs[c]); + } + float denom = 0; + for (float& p : probs) { + p = std::exp(p - max); + denom += p; + } + for (size_t j = 0; j < d; ++j) { + float value = 0; + for (size_t c = 0; c < context; ++c) { + const Tensor& cache = pv && c >= size_t(pos) ? *pv : v; + const size_t ci = pv && c >= size_t(pos) ? c - pos : c; + value += + probs[c] / denom * float(cache.values[(ci * hk + kvhead) * d + j]); + } + out.values[job * d + j] = value; + } + }); + return out; +} +static ReferenceOutputs sdpa_reference_outputs( + const Json& spec, + const TensorMap& tensors, + size_t threads) { + Tensor out = sdpa_reference(spec, tensors, threads); + ReferenceOutputs result{ + {spec.at("args").back().at("tensor").get(), + std::move(out.values)}}; + if (spec.at("operator") == "sdpa_with_kv_cache.default") { + const size_t pos = spec.at("args").at(5).at("int"); + for (const auto& pair : {std::pair{3, 1}, {4, 2}}) { + const Tensor& cache = tensor_arg(spec, pair.first, tensors); + const Tensor& projected = tensor_arg(spec, pair.second, tensors); + auto values = cache.values; + const size_t row_size = cache.sizes[2] * cache.sizes[3]; + require( + pos * row_size + projected.values.size() <= values.size(), + "SDPA projected cache update exceeds capacity"); + std::copy( + projected.values.begin(), + projected.values.end(), + values.begin() + pos * row_size); + result[spec.at("args").at(pair.first).at("tensor").get()] = + std::move(values); + } + } + return result; +} +REGISTER_REFERENCE_FUNCTIONS { + REGISTER_REFERENCE_OUTPUTS_FN(sdpa, sdpa_reference_outputs); + REGISTER_OPERATOR_REFERENCE(llama.custom_sdpa.default, sdpa); + REGISTER_OPERATOR_REFERENCE(sdpa_with_kv_cache.default, sdpa); +} +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/attention/sdpa/SdpaReference.h b/backends/vulkan/test/ops/benchmark_v2/operators/attention/sdpa/SdpaReference.h new file mode 100644 index 00000000000..cf8962baf44 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/attention/sdpa/SdpaReference.h @@ -0,0 +1,16 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include + +namespace vkcompute::shader_profiling::reference { + +Tensor +sdpa_reference(const Json& spec, const TensorMap& tensors, size_t threads); + +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/attention/sdpa/SdpaRoutes.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/attention/sdpa/SdpaRoutes.cpp new file mode 100644 index 00000000000..8df993ff582 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/attention/sdpa/SdpaRoutes.cpp @@ -0,0 +1,51 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include +#include + +namespace vkcompute::shader_profiling { +namespace { + +template +Json build_sdpa_route( + ComputeGraph& graph, + const std::vector& args, + const Json& options) { + if (!options.empty()) + throw std::invalid_argument("SDPA routes accept no options"); + if (args.size() != 9) + throw std::invalid_argument("SDPA expects nine production arguments"); + sdpa_impl_with_shader_override(graph, args, graph.add_scalar(Override)); + const char* name = Override == kShaderOverrideForceGqa ? "gqa" + : Override == kShaderOverrideForceTile2 ? "gqa_tile2" + : Override == kShaderOverrideForceBase ? "gqa_base" + : "non_gqa"; + return {{"name", name}, {"options", options}}; +} + +REGISTER_OPERATOR_ROUTES { + REGISTER_OPERATOR_ROUTE( + llama.custom_sdpa.default, + gqa, + (build_sdpa_route)); + REGISTER_OPERATOR_ROUTE( + llama.custom_sdpa.default, + non_gqa, + (build_sdpa_route)); + REGISTER_OPERATOR_ROUTE( + llama.custom_sdpa.default, + gqa_tile2, + (build_sdpa_route)); + REGISTER_OPERATOR_ROUTE( + llama.custom_sdpa.default, + gqa_base, + (build_sdpa_route)); +} + +} // namespace +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/attention/sdpa/SdpaThroughput.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/attention/sdpa/SdpaThroughput.cpp new file mode 100644 index 00000000000..018f2c441fe --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/attention/sdpa/SdpaThroughput.cpp @@ -0,0 +1,37 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +namespace vkcompute::shader_profiling { +namespace { +double sdpa_flops(const Json& spec) { + const auto q = throughput::tensor_argument(spec, 0) + .at("sizes") + .get>(); + const bool fused = spec.value("operator_name", spec.value("operator", "")) == + "sdpa_with_kv_cache.default"; + const auto k = throughput::tensor_argument(spec, fused ? 3 : 1) + .at("sizes") + .get>(); + const int64_t pos = spec.at("args").at(fused ? 5 : 3).at("int"); + const int64_t context = pos + q.at(1); + if (pos < 0 || context > k.at(1)) + throw std::invalid_argument( + "SDPA work count context exceeds cache capacity"); + return 4.0 * q.at(1) * q.at(2) * q.at(3) * context; +} +REGISTER_THROUGHPUT_FUNCTIONS { + REGISTER_OPERATOR_MEMORY_TRAFFIC( + llama.custom_sdpa.default, throughput::tensor_traffic); + REGISTER_OPERATOR_COMPUTE_FLOPS(llama.custom_sdpa.default, sdpa_flops); + REGISTER_OPERATOR_MEMORY_TRAFFIC( + sdpa_with_kv_cache.default, throughput::tensor_traffic); + REGISTER_OPERATOR_COMPUTE_FLOPS(sdpa_with_kv_cache.default, sdpa_flops); +} + +} // namespace +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/attention/sdpa/cases/correctness.json b/backends/vulkan/test/ops/benchmark_v2/operators/attention/sdpa/cases/correctness.json new file mode 100644 index 00000000000..8b2a50f06ec --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/attention/sdpa/cases/correctness.json @@ -0,0 +1,1522 @@ +[ + { + "name": "sdpa Llama-3.2-1B prefill D64 H32 Hkv8 S128 C128 P0 float16 texture3d default", + "operator": "llama.custom_sdpa.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.01, + "rtol": 0.01 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "q", + "sizes": [1, 128, 32, 64], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "k_cache", + "sizes": [1, 128, 8, 64], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "value_cache", + "sizes": [1, 128, 8, 64], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int": 0 + }, + null, + { + "float": 0.0 + }, + { + "bool": true + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 128, 32, 64], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "sdpa float16 texture3d width", + "operator": "llama.custom_sdpa.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.01, + "rtol": 0.01 + }, + "sweep_variables": { + "v_SIZES": [ + [1, 128, 16, 128], + [1, 128, 24, 128] + ] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "q", + "sizes": "v_SIZES", + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "k_cache", + "sizes": [1, 128, 8, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "value_cache", + "sizes": [1, 128, 8, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int": 0 + }, + null, + { + "float": 0.0 + }, + { + "bool": true + }, + null, + { + "tensor": { + "name": "output", + "sizes": "v_SIZES", + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "sdpa default_scale decode D64 H8 Hkv2 S1 C32 P31 float32 texture3d production", + "operator": "llama.custom_sdpa.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "q", + "sizes": [1, 1, 8, 64], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "k_cache", + "sizes": [1, 32, 2, 64], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "value_cache", + "sizes": [1, 32, 2, 64], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int": 31 + }, + null, + { + "float": 0.0 + }, + { + "bool": true + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 1, 8, 64], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "sdpa accu prefill D64 H8 Hkv2 S16 C16 P0 float32 texture3d default", + "operator": "llama.custom_sdpa.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "q", + "sizes": [1, 16, 8, 64], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "k_cache", + "sizes": [1, 16, 2, 64], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "value_cache", + "sizes": [1, 16, 2, 64], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int": 0 + }, + null, + { + "float": 0.0 + }, + { + "bool": true + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 8, 64], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "sdpa Gemma-4-E2B sliding_decode D256 H8 Hkv1 S1 C32 P31 float32 texture3d production", + "operator": "llama.custom_sdpa.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "q", + "sizes": [1, 1, 8, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "k_cache", + "sizes": [1, 32, 1, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "value_cache", + "sizes": [1, 32, 1, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int": 31 + }, + null, + { + "float": 0.0 + }, + { + "bool": true + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 1, 8, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "sdpa Gemma-4-E2B full_decode D512 H8 Hkv1 S1 C16 P15 float32 texture3d production", + "operator": "llama.custom_sdpa.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "q", + "sizes": [1, 1, 8, 512], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "k_cache", + "sizes": [1, 16, 1, 512], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "value_cache", + "sizes": [1, 16, 1, 512], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int": 15 + }, + null, + { + "float": 0.0 + }, + { + "bool": true + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 1, 8, 512], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "sdpa Gemma-4-E2B sliding_decode D256 H8 Hkv1 S1 C32 P31 float32 texture3d", + "operator": "llama.custom_sdpa.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["gqa_tile2", "gqa_base", "non_gqa"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "q", + "sizes": [1, 1, 8, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "k_cache", + "sizes": [1, 32, 1, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "value_cache", + "sizes": [1, 32, 1, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int": 31 + }, + null, + { + "float": 0.0 + }, + { + "bool": true + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 1, 8, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "sdpa Gemma-4-E2B full_decode D512 H8 Hkv1 S1 C16 P15 float32 texture3d", + "operator": "llama.custom_sdpa.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["gqa_tile2", "gqa_base", "non_gqa"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "q", + "sizes": [1, 1, 8, 512], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "k_cache", + "sizes": [1, 16, 1, 512], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "value_cache", + "sizes": [1, 16, 1, 512], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int": 15 + }, + null, + { + "float": 0.0 + }, + { + "bool": true + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 1, 8, 512], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "sdpa Gemma-4-E2B sliding_prefill D256 H8 Hkv1 S4 C4 P0 float32 texture3d fused", + "operator": "sdpa_with_kv_cache.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001, + "outputs": ["output"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "q", + "sizes": [1, 4, 8, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "k_projected", + "sizes": [1, 4, 1, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "value_projected", + "sizes": [1, 4, 1, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "k_cache", + "sizes": [1, 4, 1, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + }, + "access": "read_write" + } + }, + { + "tensor": { + "name": "value_cache", + "sizes": [1, 4, 1, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + }, + "access": "read_write" + } + }, + { + "int": 0 + }, + { + "int": 4 + }, + null, + { + "float": 0.0 + }, + { + "bool": true + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 4, 8, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "sdpa Gemma-4-E2B full_prefill D512 H8 Hkv1 S4 C4 P0 float32 texture3d fused", + "operator": "sdpa_with_kv_cache.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001, + "outputs": ["output"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "q", + "sizes": [1, 4, 8, 512], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "k_projected", + "sizes": [1, 4, 1, 512], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "value_projected", + "sizes": [1, 4, 1, 512], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "k_cache", + "sizes": [1, 4, 1, 512], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + }, + "access": "read_write" + } + }, + { + "tensor": { + "name": "value_cache", + "sizes": [1, 4, 1, 512], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + }, + "access": "read_write" + } + }, + { + "int": 0 + }, + { + "int": 4 + }, + null, + { + "float": 0.0 + }, + { + "bool": true + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 4, 8, 512], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "sdpa Gemma-4-E2B sliding_prefill_real D256 H8 Hkv1 S18 C512 P0 float32 texture3d fused", + "operator": "sdpa_with_kv_cache.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001, + "outputs": ["output"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "q", + "sizes": [1, 18, 8, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "k_projected", + "sizes": [1, 18, 1, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "value_projected", + "sizes": [1, 18, 1, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "k_cache", + "sizes": [1, 512, 1, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + }, + "access": "read_write" + } + }, + { + "tensor": { + "name": "value_cache", + "sizes": [1, 512, 1, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + }, + "access": "read_write" + } + }, + { + "int": 0 + }, + { + "int": 18 + }, + null, + { + "float": 0.0 + }, + { + "bool": true + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 18, 8, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "sdpa Gemma-4-E2B full_prefill_real D512 H8 Hkv1 S18 C512 P0 float32 texture3d fused", + "operator": "sdpa_with_kv_cache.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001, + "outputs": ["output"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "q", + "sizes": [1, 18, 8, 512], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "k_projected", + "sizes": [1, 18, 1, 512], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "value_projected", + "sizes": [1, 18, 1, 512], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "k_cache", + "sizes": [1, 512, 1, 512], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + }, + "access": "read_write" + } + }, + { + "tensor": { + "name": "value_cache", + "sizes": [1, 512, 1, 512], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + }, + "access": "read_write" + } + }, + { + "int": 0 + }, + { + "int": 18 + }, + null, + { + "float": 0.0 + }, + { + "bool": true + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 18, 8, 512], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "sdpa Gemma-4-E2B sliding_decode_prefix D256 H8 Hkv1 S1 C512 P17 float32 texture3d fused", + "operator": "sdpa_with_kv_cache.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001, + "outputs": ["output"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "q", + "sizes": [1, 1, 8, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "k_projected", + "sizes": [1, 1, 1, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "value_projected", + "sizes": [1, 1, 1, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "k_cache", + "sizes": [1, 512, 1, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + }, + "access": "read_write" + } + }, + { + "tensor": { + "name": "value_cache", + "sizes": [1, 512, 1, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + }, + "access": "read_write" + } + }, + { + "int": 17 + }, + { + "int": 1 + }, + null, + { + "float": 0.0 + }, + { + "bool": true + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 1, 8, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "sdpa Gemma-4-E2B full_decode_prefix D512 H8 Hkv1 S1 C512 P17 float32 texture3d fused", + "operator": "sdpa_with_kv_cache.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001, + "outputs": ["output"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "q", + "sizes": [1, 1, 8, 512], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "k_projected", + "sizes": [1, 1, 1, 512], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "value_projected", + "sizes": [1, 1, 1, 512], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "k_cache", + "sizes": [1, 512, 1, 512], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + }, + "access": "read_write" + } + }, + { + "tensor": { + "name": "value_cache", + "sizes": [1, 512, 1, 512], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + }, + "access": "read_write" + } + }, + { + "int": 17 + }, + { + "int": 1 + }, + null, + { + "float": 0.0 + }, + { + "bool": true + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 1, 8, 512], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "sdpa float32 width", + "operator": "llama.custom_sdpa.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["gqa", "non_gqa", "gqa_tile2"], + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "q", + "sizes": [1, 1, 8, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "k_cache", + "sizes": [1, 32, 2, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "value_cache", + "sizes": [1, 32, 2, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int": 31 + }, + null, + { + "float": 0.0 + }, + { + "bool": true + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 1, 8, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "sdpa float32 width (15)", + "operator": "llama.custom_sdpa.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["gqa", "non_gqa", "gqa_tile2"], + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "q", + "sizes": [1, 1, 8, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "k_cache", + "sizes": [1, 32, 2, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "value_cache", + "sizes": [1, 32, 2, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int": 31 + }, + null, + { + "float": 0.0 + }, + { + "bool": true + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 1, 8, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "sdpa accu_tile2_d4 decode D4 H8 Hkv2 S1 C32 P31 float32", + "operator": "llama.custom_sdpa.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "gqa_tile2", + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "q", + "sizes": [1, 1, 8, 4], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "k_cache", + "sizes": [1, 32, 2, 4], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "value_cache", + "sizes": [1, 32, 2, 4], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int": 31 + }, + null, + { + "float": 0.0 + }, + { + "bool": true + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 1, 8, 4], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/attention/sdpa/cases/decode.json b/backends/vulkan/test/ops/benchmark_v2/operators/attention/sdpa/cases/decode.json new file mode 100644 index 00000000000..b3b5d355a0a --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/attention/sdpa/cases/decode.json @@ -0,0 +1,494 @@ +[ + { + "name": "sdpa Llama-3.2-1B decode D64 H32 Hkv8 S1 C512 P511 float16 texture3d", + "operator": "llama.custom_sdpa.default", + "target_ms": 100, + "warmup_runs": 10, + "benchmark_runs": 30, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.01, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["gqa", "non_gqa"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "q", + "sizes": [1, 1, 32, 64], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "k_cache", + "sizes": [1, 512, 8, 64], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "value_cache", + "sizes": [1, 512, 8, 64], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int": 511 + }, + null, + { + "float": 0.0 + }, + { + "bool": true + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 1, 32, 64], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "sdpa", + "operator": "llama.custom_sdpa.default", + "target_ms": 100, + "warmup_runs": 10, + "benchmark_runs": 30, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.01, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["gqa", "non_gqa"], + "v_SIZES": [ + [1, 1, 16, 128], + [1, 1, 24, 128] + ] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "q", + "sizes": "v_SIZES", + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "k_cache", + "sizes": [1, 512, 8, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "value_cache", + "sizes": [1, 512, 8, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int": 511 + }, + null, + { + "float": 0.0 + }, + { + "bool": true + }, + null, + { + "tensor": { + "name": "output", + "sizes": "v_SIZES", + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "sdpa Llama-3.2-1B decode D64 H32 Hkv8 S1 C1024 P1023 float16 texture3d", + "operator": "llama.custom_sdpa.default", + "target_ms": 100, + "warmup_runs": 10, + "benchmark_runs": 30, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.01, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["gqa", "non_gqa"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "q", + "sizes": [1, 1, 32, 64], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "k_cache", + "sizes": [1, 1024, 8, 64], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "value_cache", + "sizes": [1, 1024, 8, 64], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int": 1023 + }, + null, + { + "float": 0.0 + }, + { + "bool": true + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 1, 32, 64], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "sdpa (4)", + "operator": "llama.custom_sdpa.default", + "target_ms": 100, + "warmup_runs": 10, + "benchmark_runs": 30, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.01, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["gqa", "non_gqa"], + "v_SIZES": [ + [1, 1, 16, 128], + [1, 1, 24, 128] + ] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "q", + "sizes": "v_SIZES", + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "k_cache", + "sizes": [1, 1024, 8, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "value_cache", + "sizes": [1, 1024, 8, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int": 1023 + }, + null, + { + "float": 0.0 + }, + { + "bool": true + }, + null, + { + "tensor": { + "name": "output", + "sizes": "v_SIZES", + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "sdpa Llama-3.2-1B decode D64 H32 Hkv8 S1 C4096 P4095 float16 texture3d", + "operator": "llama.custom_sdpa.default", + "target_ms": 100, + "warmup_runs": 10, + "benchmark_runs": 30, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.01, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["gqa", "non_gqa"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "q", + "sizes": [1, 1, 32, 64], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "k_cache", + "sizes": [1, 4096, 8, 64], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "value_cache", + "sizes": [1, 4096, 8, 64], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int": 4095 + }, + null, + { + "float": 0.0 + }, + { + "bool": true + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 1, 32, 64], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "sdpa (6)", + "operator": "llama.custom_sdpa.default", + "target_ms": 100, + "warmup_runs": 10, + "benchmark_runs": 30, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.01, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["gqa", "non_gqa"], + "v_SIZES": [ + [1, 1, 16, 128], + [1, 1, 24, 128] + ] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "q", + "sizes": "v_SIZES", + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "k_cache", + "sizes": [1, 4096, 8, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "value_cache", + "sizes": [1, 4096, 8, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int": 4095 + }, + null, + { + "float": 0.0 + }, + { + "bool": true + }, + null, + { + "tensor": { + "name": "output", + "sizes": "v_SIZES", + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/attention/sdpa/cases/performance.json b/backends/vulkan/test/ops/benchmark_v2/operators/attention/sdpa/cases/performance.json new file mode 100644 index 00000000000..04392ffce6e --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/attention/sdpa/cases/performance.json @@ -0,0 +1,160 @@ +[ + { + "name": "sdpa Llama-3.2-1B prefill D64 H32 Hkv8 S512 C512 P0 float16 texture3d default", + "operator": "llama.custom_sdpa.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.01, + "rtol": 0.01 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "q", + "sizes": [1, 512, 32, 64], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "k_cache", + "sizes": [1, 512, 8, 64], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "value_cache", + "sizes": [1, 512, 8, 64], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int": 0 + }, + null, + { + "float": 0.0 + }, + { + "bool": true + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 512, 32, 64], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "sdpa", + "operator": "llama.custom_sdpa.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.01, + "rtol": 0.01 + }, + "sweep_variables": { + "v_SIZES": [ + [1, 512, 16, 128], + [1, 512, 24, 128] + ] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "q", + "sizes": "v_SIZES", + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "k_cache", + "sizes": [1, 512, 8, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "value_cache", + "sizes": [1, 512, 8, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int": 0 + }, + null, + { + "float": 0.0 + }, + { + "bool": true + }, + null, + { + "tensor": { + "name": "output", + "sizes": "v_SIZES", + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/binary/BinaryReference.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/binary/BinaryReference.cpp new file mode 100644 index 00000000000..11c6c870c92 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/binary/BinaryReference.cpp @@ -0,0 +1,111 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include + +#include + +namespace vkcompute::shader_profiling::reference { + +Tensor binary_reference( + const Json& spec, + const TensorMap& tensors, + const std::string& function, + size_t threads) { + std::vector bindings; + for (const Json& binding : spec.at("bindings")) { + if (binding.contains("tensor")) { + bindings.push_back(binding.at("tensor")); + } + } + require( + bindings.size() == (function == "copy" ? 2u : 3u), + "Copy/add reference requires output followed by tensor input bindings"); + const Tensor& a = tensors.at(bindings[1]); + const auto sizes = spec.at("tensors") + .at(bindings[0]) + .at("sizes") + .get>(); + if (function == "copy") { + require(a.sizes == sizes, "Copy input/output shapes must match"); + return a; + } + const Tensor& b = tensors.at(bindings[2]); + const size_t rank = std::max(a.sizes.size(), b.sizes.size()); + std::vector output_sizes(rank, 1); + std::vector a_strides(rank), b_strides(rank); + size_t a_stride = 1; + size_t b_stride = 1; + for (size_t dim = rank; dim > 0; --dim) { + const size_t i = dim - 1; + const int64_t a_size = dim > rank - a.sizes.size() + ? a.sizes[dim - (rank - a.sizes.size()) - 1] + : 1; + const int64_t b_size = dim > rank - b.sizes.size() + ? b.sizes[dim - (rank - b.sizes.size()) - 1] + : 1; + require( + a_size == b_size || a_size == 1 || b_size == 1, + "Addition input shapes do not broadcast"); + output_sizes[i] = std::max(a_size, b_size); + a_strides[i] = a_size == 1 ? 0 : a_stride; + b_strides[i] = b_size == 1 ? 0 : b_stride; + a_stride *= a_size; + b_stride *= b_size; + } + require( + output_sizes == sizes, + "Addition output shape does not match broadcast shape"); + double alpha = 1; + if (spec.contains("push_constants") && !spec.at("push_constants").empty()) { + alpha = spec.at("push_constants").at(0).at("data").at(0); + } + Tensor output{sizes, std::vector(tensor_numel(sizes))}; + constexpr size_t kChunk = 4096; + parallel_for( + (output.values.size() + kChunk - 1) / kChunk, threads, [&](size_t chunk) { + const size_t end = std::min(output.values.size(), (chunk + 1) * kChunk); + for (size_t index = chunk * kChunk; index < end; ++index) { + size_t remainder = index; + size_t ai = 0; + size_t bi = 0; + for (size_t dim = rank; dim > 0; --dim) { + const size_t coordinate = remainder % output_sizes[dim - 1]; + remainder /= output_sizes[dim - 1]; + ai += coordinate * a_strides[dim - 1]; + bi += coordinate * b_strides[dim - 1]; + } + output.values[index] = a.values[ai] + alpha * b.values[bi]; + } + }); + return output; +} + +namespace { + +Tensor +copy_reference(const Json& spec, const TensorMap& tensors, size_t threads) { + return binary_reference(spec, tensors, "copy", threads); +} + +Tensor +add_reference(const Json& spec, const TensorMap& tensors, size_t threads) { + return binary_reference(spec, tensors, "add", threads); +} + +} // namespace + +REGISTER_REFERENCE_FUNCTIONS { + REGISTER_REFERENCE_FN(copy, copy_reference); + REGISTER_REFERENCE_FN(add, add_reference); + REGISTER_SHADER_REFERENCE(buffer_to_buffer_float, copy); + REGISTER_SHADER_REFERENCE(buffer_to_buffer_half, copy); + REGISTER_SHADER_REFERENCE(binary_add_texture3d_float, add); + REGISTER_SHADER_REFERENCE(binary_add_texture3d_half, add); +} + +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/binary/BinaryReference.h b/backends/vulkan/test/ops/benchmark_v2/operators/binary/BinaryReference.h new file mode 100644 index 00000000000..ebc2a53042e --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/binary/BinaryReference.h @@ -0,0 +1,19 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include + +namespace vkcompute::shader_profiling::reference { + +Tensor binary_reference( + const Json& spec, + const TensorMap& tensors, + const std::string& function, + size_t threads); + +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/binary/BinaryThroughput.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/binary/BinaryThroughput.cpp new file mode 100644 index 00000000000..15295080587 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/binary/BinaryThroughput.cpp @@ -0,0 +1,29 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +namespace vkcompute::shader_profiling { +namespace { + +double copy_flops(const Json&) { + return 0; +} + +double add_flops(const Json& spec) { + return 2 * throughput::tensor_elements(throughput::output_tensor(spec)); +} + +} // namespace + +REGISTER_THROUGHPUT_FUNCTIONS { + REGISTER_SHADER_MEMORY_TRAFFIC(buffer_to_buffer, throughput::tensor_traffic); + REGISTER_SHADER_COMPUTE_FLOPS(buffer_to_buffer, copy_flops); + REGISTER_SHADER_MEMORY_TRAFFIC(binary_add, throughput::tensor_traffic); + REGISTER_SHADER_COMPUTE_FLOPS(binary_add, add_flops); +} + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/binary/add/AddReference.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/binary/add/AddReference.cpp new file mode 100644 index 00000000000..c9d0fc165df --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/binary/add/AddReference.cpp @@ -0,0 +1,30 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include +#include + +namespace vkcompute::shader_profiling::reference { +Tensor add_tensor_reference( + const Json& spec, + const TensorMap& tensors, + size_t threads) { + Json shader_spec = spec; + shader_spec["bindings"] = { + spec.at("args").back(), spec.at("args").at(0), spec.at("args").at(1)}; + const Json& alpha = spec.at("args").at(2); + const double alpha_value = alpha.is_null() ? 1.0 + : alpha.contains("float") ? alpha.at("float").get() + : alpha.at("int").get(); + shader_spec["push_constants"] = {{{"data", {alpha_value}}}}; + return binary_reference(shader_spec, tensors, "add", threads); +} +REGISTER_REFERENCE_FUNCTIONS { + REGISTER_REFERENCE_FN(add_tensor, add_tensor_reference); + REGISTER_OPERATOR_REFERENCE(aten.add.Tensor, add_tensor); +} +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/binary/add/AddReference.h b/backends/vulkan/test/ops/benchmark_v2/operators/binary/add/AddReference.h new file mode 100644 index 00000000000..212dfd621fd --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/binary/add/AddReference.h @@ -0,0 +1,18 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include + +namespace vkcompute::shader_profiling::reference { + +Tensor add_tensor_reference( + const Json& spec, + const TensorMap& tensors, + size_t threads); + +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/binary/add/AddThroughput.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/binary/add/AddThroughput.cpp new file mode 100644 index 00000000000..8cc7d7de64a --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/binary/add/AddThroughput.cpp @@ -0,0 +1,19 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +namespace vkcompute::shader_profiling { +namespace { +double add_tensor_flops(const Json& spec) { + return 2 * throughput::tensor_elements(throughput::output_tensor(spec)); +} +REGISTER_THROUGHPUT_FUNCTIONS { + REGISTER_OPERATOR_MEMORY_TRAFFIC(aten.add.Tensor, throughput::tensor_traffic); + REGISTER_OPERATOR_COMPUTE_FLOPS(aten.add.Tensor, add_tensor_flops); +} +} // namespace +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/binary/add/cases/correctness.json b/backends/vulkan/test/ops/benchmark_v2/operators/binary/add/cases/correctness.json new file mode 100644 index 00000000000..175fe082aa9 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/binary/add/cases/correctness.json @@ -0,0 +1,72 @@ +[ + { + "name": "add correctness", + "operator": "aten.add.Tensor", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_DTYPE": ["float32", "float16"], + "v_SIZES": [ + [1, 64, 64], + [1, 128, 128], + [1, 256, 256], + [1, 512, 512], + [1, 1, 1024], + [1, 1024, 1], + [32, 32, 32], + [16, 128, 64] + ], + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "a", + "sizes": "v_SIZES", + "dtype": "v_DTYPE", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [1, 1] + } + } + }, + { + "tensor": { + "name": "b", + "sizes": "v_SIZES", + "dtype": "v_DTYPE", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [1, 1] + } + } + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": "v_SIZES", + "dtype": "v_DTYPE", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/binary/q8ta/Q8taBinaryReference.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/binary/q8ta/Q8taBinaryReference.cpp new file mode 100644 index 00000000000..bdd24d4aaf3 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/binary/q8ta/Q8taBinaryReference.cpp @@ -0,0 +1,35 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include + +namespace vkcompute::shader_profiling::reference { +static Tensor +q8ta_add_reference(const Json& spec, const TensorMap& tensors, size_t threads) { + const Tensor& a = tensor_arg(spec, 0, tensors); + const Tensor& b = tensor_arg(spec, 1, tensors); + Tensor out = q8ta::output_tensor(spec, spec.at("args").size() - 1); + const float as = q8ta::scale(spec, 2), bs = q8ta::scale(spec, 4), + os = q8ta::scale(spec, 6), alpha = q8ta::scale(spec, 8); + const int64_t az = q8ta::integer(spec, 3), bz = q8ta::integer(spec, 5), + oz = q8ta::integer(spec, 7); + parallel_for(out.values.size(), threads, [&](size_t i) { + const int64_t qa = + int64_t(a.values[q8ta::broadcast_index(i, out.sizes, a.sizes)]); + const int64_t qb = + int64_t(b.values[q8ta::broadcast_index(i, out.sizes, b.sizes)]); + const float value = + q8ta::dequantize(qa, as, az) + alpha * q8ta::dequantize(qb, bs, bz); + out.values[i] = double(q8ta::quantize(value, os, oz)); + }); + return out; +} +REGISTER_REFERENCE_FUNCTIONS { + REGISTER_REFERENCE_FN(q8ta_add, q8ta_add_reference); + REGISTER_OPERATOR_REFERENCE(et_vk.q8ta_add.default, q8ta_add); +} +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/binary/q8ta/Q8taBinaryThroughput.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/binary/q8ta/Q8taBinaryThroughput.cpp new file mode 100644 index 00000000000..462011239ee --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/binary/q8ta/Q8taBinaryThroughput.cpp @@ -0,0 +1,20 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +namespace vkcompute::shader_profiling { +namespace { +double q8ta_add_flops(const Json& spec) { + return 6 * throughput::tensor_elements(throughput::output_tensor(spec)); +} +REGISTER_THROUGHPUT_FUNCTIONS { + REGISTER_OPERATOR_MEMORY_TRAFFIC( + et_vk.q8ta_add.default, throughput::tensor_traffic); + REGISTER_OPERATOR_COMPUTE_FLOPS(et_vk.q8ta_add.default, q8ta_add_flops); +} +} // namespace +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/binary/q8ta/cases/correctness.json b/backends/vulkan/test/ops/benchmark_v2/operators/binary/q8ta/cases/correctness.json new file mode 100644 index 00000000000..eefa059c49b --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/binary/q8ta/cases/correctness.json @@ -0,0 +1,4087 @@ +[ + { + "name": "q8ta/binary/correctness", + "operator": "et_vk.q8ta_add.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0063751008299493, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [144], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [144], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [144], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [144], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [144], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [144], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/binary/correctness (2)", + "operator": "et_vk.q8ta_add.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0063751008299493, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [90], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [90], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [90], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [90], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [90], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [90], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/binary/correctness (3)", + "operator": "et_vk.q8ta_add.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0063751008299493, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"], + "v_OTHER_SIZES": [[1, 16, 32], [1, 16, 1], [32]] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": "v_OTHER_SIZES", + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": "v_OTHER_SIZES", + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/binary/correctness (4)", + "operator": "et_vk.q8ta_add.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0063751008299493, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 3, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 3, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 3, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 3, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 3, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 3, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/binary/correctness (5)", + "operator": "et_vk.q8ta_add.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0063751008299493, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 144], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 144], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 144], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 144], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 144], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 144], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/binary/correctness (6)", + "operator": "et_vk.q8ta_add.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0063751008299493, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 90], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 90], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 90], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 90], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 90], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 90], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/binary/correctness (7)", + "operator": "et_vk.q8ta_add.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0063751008299493, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 4], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 4], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 4], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 4], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 4], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 4], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/binary/correctness (8)", + "operator": "et_vk.q8ta_add.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0063751008299493, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [3, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [3, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [3, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [3, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [3, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [3, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/binary/correctness (9)", + "operator": "et_vk.q8ta_add.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0063751008299493, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 3, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 3, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 3, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 3, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 3, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 3, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/binary/correctness (10)", + "operator": "et_vk.q8ta_add.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0063751008299493, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 8, 32, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 8, 32, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 8, 32, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 8, 32, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 8, 32, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 8, 32, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/binary/correctness (11)", + "operator": "et_vk.q8ta_add.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0063751008299493, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 24, 24], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 16, 24, 24], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 24, 24], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 24, 24], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 16, 24, 24], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 24, 24], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/binary/correctness (12)", + "operator": "et_vk.q8ta_add.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0063751008299493, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 12, 12], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 32, 12, 12], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 12, 12], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 12, 12], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 32, 12, 12], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 12, 12], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/binary/correctness (13)", + "operator": "et_vk.q8ta_add.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0063751008299493, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 1, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 1, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 1, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 1, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 1, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 1, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/binary/correctness (14)", + "operator": "et_vk.q8ta_add.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0063751008299493, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 3, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 3, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 3, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 3, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 3, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 3, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/binary/correctness (15)", + "operator": "et_vk.q8ta_add.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0063751008299493, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 4, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 4, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 4, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 4, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 4, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 4, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/binary/correctness (16)", + "operator": "et_vk.q8ta_add.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0063751008299493, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 8, 20, 20], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 8, 20, 20], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 8, 20, 20], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 8, 20, 20], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 8, 20, 20], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 8, 20, 20], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/binary/correctness (17)", + "operator": "et_vk.q8ta_add.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0063751008299493, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 14, 14], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 16, 14, 14], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 14, 14], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 14, 14], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 16, 14, 14], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 14, 14], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/binary/correctness (18)", + "operator": "et_vk.q8ta_add.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0063751008299493, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 8, 28, 28], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 8, 28, 28], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 8, 28, 28], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 8, 28, 28], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 8, 28, 28], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 8, 28, 28], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/binary/correctness (19)", + "operator": "et_vk.q8ta_add.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0063751008299493, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 3, 15, 15], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 3, 15, 15], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 3, 15, 15], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 3, 15, 15], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 3, 15, 15], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 3, 15, 15], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/binary/correctness (20)", + "operator": "et_vk.q8ta_add.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0063751008299493, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 13, 31, 31], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 13, 31, 31], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 13, 31, 31], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 13, 31, 31], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 13, 31, 31], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 13, 31, 31], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/binary/correctness (21)", + "operator": "et_vk.q8ta_add.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0063751008299493, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 17, 23, 23], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 17, 23, 23], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 17, 23, 23], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 17, 23, 23], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 17, 23, 23], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 17, 23, 23], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/binary/correctness (22)", + "operator": "et_vk.q8ta_add.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0063751008299493, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 64, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 64, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/binary/correctness (23)", + "operator": "et_vk.q8ta_add.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0063751008299493, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 32, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 32, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/binary/correctness (24)", + "operator": "et_vk.q8ta_add.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0063751008299493, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128, 56, 56], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 128, 56, 56], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 128, 56, 56], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128, 56, 56], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 128, 56, 56], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 128, 56, 56], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/binary/correctness (25)", + "operator": "et_vk.q8ta_add.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0063751008299493, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 128, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 128, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 128, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 128, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/binary/correctness (26)", + "operator": "et_vk.q8ta_add.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0063751008299493, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 60, 512], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 1, 512], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 60, 512], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 60, 512], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 1, 512], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 60, 512], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/binary/correctness (27)", + "operator": "et_vk.q8ta_add.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0063751008299493, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 1, 512], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 60, 512], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 60, 512], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 1, 512], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 60, 512], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 60, 512], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/binary/correctness (28)", + "operator": "et_vk.q8ta_add.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0063751008299493, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"], + "v_OTHER_SIZES": [ + [1, 8, 1, 1], + [1, 1, 16, 16] + ] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 8, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": "v_OTHER_SIZES", + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 8, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 8, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": "v_OTHER_SIZES", + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 8, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/binary/correctness (29)", + "operator": "et_vk.q8ta_add.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0063751008299493, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [2, 8, 6, 6], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 8, 6, 6], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [2, 8, 6, 6], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [2, 8, 6, 6], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 8, 6, 6], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [2, 8, 6, 6], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/binary/correctness (30)", + "operator": "et_vk.q8ta_add.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0063751008299493, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 13, 7, 9], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 1, 7, 1], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 13, 7, 9], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 13, 7, 9], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 1, 7, 1], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 13, 7, 9], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/binary/q8ta/cases/debug.json b/backends/vulkan/test/ops/benchmark_v2/operators/binary/q8ta/cases/debug.json new file mode 100644 index 00000000000..9b4fa9d73fa --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/binary/q8ta/cases/debug.json @@ -0,0 +1,483 @@ +[ + { + "name": "q8ta/binary/debug", + "operator": "et_vk.q8ta_add.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0063751008299493, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 16, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 16, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/binary/debug (2)", + "operator": "et_vk.q8ta_add.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0063751008299493, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 144], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 144], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 144], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 144], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 144], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 144], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/binary/debug (3)", + "operator": "et_vk.q8ta_add.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0063751008299493, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 90], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 90], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 90], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 90], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "other", + "sizes": [1, 90], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.007842999882996082 + }, + { + "int": 3 + }, + { + "float": 0.009411999955773354 + }, + { + "int": -2 + }, + { + "float": 0.015685999765992165 + }, + { + "int": 1 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 90], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta add alpha", + "operator": "et_vk.q8ta_add.default", + "target_ms": 10, + "benchmark_runs": 5, + "warmup_runs": 1, + "max_dispatches": 512, + "validation_method": { + "function": "is_close", + "atol": 0, + "rtol": 0 + }, + "sweep_variables": { + "v_ARG8_FLOAT": [0, 0.5, -1] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [2, 1], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w", + "data": [120, -120] + } + }, + { + "tensor": { + "name": "other", + "sizes": [2], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w", + "data": [-20, 20] + } + }, + { + "float": 1 + }, + { + "int": 0 + }, + { + "float": 1 + }, + { + "int": 0 + }, + { + "float": 1 + }, + { + "int": 0 + }, + { + "float": "v_ARG8_FLOAT" + }, + { + "tensor": { + "name": "output", + "sizes": [2, 2], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/clone/q8ta/Q8taCloneReference.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/clone/q8ta/Q8taCloneReference.cpp new file mode 100644 index 00000000000..6bb270cc4fe --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/clone/q8ta/Q8taCloneReference.cpp @@ -0,0 +1,18 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +namespace vkcompute::shader_profiling::reference { +static Tensor +clone_reference(const Json& spec, const TensorMap& tensors, size_t) { + return tensor_arg(spec, 0, tensors); +} +REGISTER_REFERENCE_FUNCTIONS { + REGISTER_REFERENCE_FN(clone, clone_reference); + REGISTER_OPERATOR_REFERENCE(aten.clone.default, clone); +} +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/clone/q8ta/Q8taCloneThroughput.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/clone/q8ta/Q8taCloneThroughput.cpp new file mode 100644 index 00000000000..2befab916a5 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/clone/q8ta/Q8taCloneThroughput.cpp @@ -0,0 +1,20 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +namespace vkcompute::shader_profiling { +namespace { +double clone_flops(const Json&) { + return 0; +} +REGISTER_THROUGHPUT_FUNCTIONS { + REGISTER_OPERATOR_MEMORY_TRAFFIC( + aten.clone.default, throughput::tensor_traffic); + REGISTER_OPERATOR_COMPUTE_FLOPS(aten.clone.default, clone_flops); +} +} // namespace +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/clone/q8ta/cases/correctness.json b/backends/vulkan/test/ops/benchmark_v2/operators/clone/q8ta/cases/correctness.json new file mode 100644 index 00000000000..bbec31650b8 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/clone/q8ta/cases/correctness.json @@ -0,0 +1,94 @@ +[ + { + "name": "q8ta/clone/correctness", + "operator": "aten.clone.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.014060718249784, + "rtol": 0 + }, + "sweep_variables": { + "v_INPUT_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"], + "v_SIZES": [ + [1, 3, 16, 16], + [1, 8, 32, 32], + [1, 16, 24, 24], + [1, 32, 12, 12], + [1, 1, 64, 64], + [1, 3, 64, 64], + [1, 4, 16, 16], + [1, 8, 20, 20], + [1, 16, 14, 14], + [1, 8, 28, 28], + [1, 3, 15, 15], + [1, 13, 31, 31], + [1, 17, 23, 23], + [1, 64, 128, 128], + [1, 32, 64, 64], + [1, 128, 56, 56], + [1, 128, 128, 128] + ], + "v_OUTPUT_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": "v_SIZES", + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_INPUT_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": "v_SIZES", + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_OUTPUT_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": "v_SIZES", + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_INPUT_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": "v_SIZES", + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_OUTPUT_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/clone/q8ta/cases/debug.json b/backends/vulkan/test/ops/benchmark_v2/operators/clone/q8ta/cases/debug.json new file mode 100644 index 00000000000..7b9ff06c23e --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/clone/q8ta/cases/debug.json @@ -0,0 +1,74 @@ +[ + { + "name": "q8ta/clone/debug", + "operator": "aten.clone.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.014060718249784, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dReference.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dReference.cpp new file mode 100644 index 00000000000..df865939107 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dReference.cpp @@ -0,0 +1,68 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include +#include +#include + +#include + +namespace vkcompute::shader_profiling::reference { + +Tensor +conv2d_reference(const Json& spec, const TensorMap& tensors, size_t threads) { + if (tensor_arg(spec, 0, tensors).sizes.size() == 3) { + return conv1d_reference(spec, tensors, threads); + } + require( + !spec.at("args").at(6).at("bool").get(), + "Conv2d reference does not support transposed convolution"); + const Tensor& input = tensor_arg(spec, 0, tensors); + const Tensor& weight = tensor_arg(spec, 1, tensors); + require(weight.sizes.size() == 4, "Conv2d reference requires rank-4 weight"); + const Conv2dParameters p{ + {weight.sizes[2], weight.sizes[3]}, + conv2d_pair_arg(spec, 3), + conv2d_pair_arg(spec, 4), + conv2d_pair_arg(spec, 5), + spec.at("args").at(8).at("int")}; + const Tensor* bias = + spec.at("args").at(2).is_null() ? nullptr : &tensor_arg(spec, 2, tensors); + const auto g = conv2d_geometry(input, weight, bias, p); + require( + weight.sizes == + std::vector( + {g.out_channels, + g.in_channels_per_group, + p.kernel[0], + p.kernel[1]}), + "Conv2d weight shape must be [Cout, Cin/groups, Kh, Kw]"); + return convolve( + input, + weight, + p, + g, + threads, + [&](int64_t oc, int64_t ci, int64_t ky, int64_t kx) { + return ((oc * g.in_channels_per_group + ci) * p.kernel[0] + ky) * + p.kernel[1] + + kx; + }, + [](float sum, double value, double weight_value) { + return std::fma(float(value), float(weight_value), sum); + }, + [&](float sum, int64_t oc) { + return sum + (bias ? float(bias->values[oc]) : 0.0f); + }); +} + +REGISTER_REFERENCE_FUNCTIONS { + REGISTER_REFERENCE_FN(conv2d, conv2d_reference); + REGISTER_OPERATOR_REFERENCE(aten.convolution.default, conv2d); +} + +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dReference.h b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dReference.h new file mode 100644 index 00000000000..15e8d9cd418 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dReference.h @@ -0,0 +1,15 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include + +namespace vkcompute::shader_profiling::reference { + +Tensor +conv2d_reference(const Json& spec, const TensorMap& tensors, size_t threads); +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dReferenceCommon.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dReferenceCommon.cpp new file mode 100644 index 00000000000..6e48de9cfa9 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dReferenceCommon.cpp @@ -0,0 +1,58 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +namespace vkcompute::shader_profiling::reference { + +std::array conv2d_pair_arg(const Json& spec, size_t index) { + const auto values = + spec.at("args").at(index).at("int_list").get>(); + require(values.size() == 2, "Conv2d spatial parameters require two values"); + return {values[0], values[1]}; +} + +Conv2dGeometry conv2d_geometry( + const Tensor& input, + const Tensor& weight, + const Tensor* bias, + const Conv2dParameters& p) { + require( + input.sizes.size() == 4 && !weight.sizes.empty(), + "Conv2d requires NCHW input"); + const int64_t channels = input.sizes[1]; + const int64_t out_channels = weight.sizes[0]; + require( + p.groups > 0 && channels % p.groups == 0 && out_channels % p.groups == 0, + "Conv2d channels must be divisible by groups"); + for (size_t dim = 0; dim < 2; ++dim) { + require( + p.kernel[dim] > 0 && p.stride[dim] > 0 && p.dilation[dim] > 0 && + p.padding[dim] >= 0, + "Conv2d requires positive kernel/stride/dilation and nonnegative padding"); + } + require( + !bias || bias->values.size() == size_t(out_channels), + "Conv2d bias size must equal output channels"); + const int64_t hn = + input.sizes[2] + 2 * p.padding[0] - p.dilation[0] * (p.kernel[0] - 1) - 1; + const int64_t wn = + input.sizes[3] + 2 * p.padding[1] - p.dilation[1] * (p.kernel[1] - 1) - 1; + require(hn >= 0 && wn >= 0, "Conv2d kernel exceeds padded input"); + return { + input.sizes[0], + channels, + input.sizes[2], + input.sizes[3], + out_channels, + channels / p.groups, + out_channels / p.groups, + channels / p.groups * p.kernel[0] * p.kernel[1], + hn / p.stride[0] + 1, + wn / p.stride[1] + 1}; +} + +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dReferenceCommon.h b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dReferenceCommon.h new file mode 100644 index 00000000000..6dbae353c5d --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dReferenceCommon.h @@ -0,0 +1,97 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include + +#include + +namespace vkcompute::shader_profiling::reference { + +struct Conv2dParameters { + std::array kernel; + std::array stride; + std::array padding; + std::array dilation; + int64_t groups; +}; + +struct Conv2dGeometry { + int64_t batches; + int64_t channels; + int64_t height; + int64_t width; + int64_t out_channels; + int64_t in_channels_per_group; + int64_t out_channels_per_group; + int64_t row_size; + int64_t out_height; + int64_t out_width; +}; + +std::array conv2d_pair_arg(const Json& spec, size_t index); +Conv2dGeometry conv2d_geometry( + const Tensor& input, + const Tensor& weight, + const Tensor* bias, + const Conv2dParameters& p); + +template < + typename Accumulator, + typename WeightIndex, + typename Accumulate, + typename Finish> +Tensor convolve( + const Tensor& input, + const Tensor& weight, + const Conv2dParameters& p, + const Conv2dGeometry& g, + size_t threads, + WeightIndex weight_index, + Accumulate accumulate, + Finish finish) { + Tensor output{ + {g.batches, g.out_channels, g.out_height, g.out_width}, + std::vector( + size_t(g.batches * g.out_channels * g.out_height * g.out_width))}; + const size_t rows = size_t(g.batches * g.out_channels * g.out_height); + parallel_for(rows, threads, [&](size_t row) { + const int64_t oy = row % g.out_height; + const int64_t oc = (row / g.out_height) % g.out_channels; + const int64_t batch = row / (g.out_height * g.out_channels); + const int64_t input_channel_start = + (oc / g.out_channels_per_group) * g.in_channels_per_group; + for (int64_t ox = 0; ox < g.out_width; ++ox) { + Accumulator sum = 0; + for (int64_t ci = 0; ci < g.in_channels_per_group; ++ci) { + const int64_t input_channel_offset = + (batch * g.channels + input_channel_start + ci) * g.height * + g.width; + for (int64_t ky = 0; ky < p.kernel[0]; ++ky) { + const int64_t iy = + oy * p.stride[0] - p.padding[0] + ky * p.dilation[0]; + if (iy < 0 || iy >= g.height) + continue; + for (int64_t kx = 0; kx < p.kernel[1]; ++kx) { + const int64_t ix = + ox * p.stride[1] - p.padding[1] + kx * p.dilation[1]; + if (ix < 0 || ix >= g.width) + continue; + sum = accumulate( + sum, + input.values[input_channel_offset + iy * g.width + ix], + weight.values[weight_index(oc, ci, ky, kx)]); + } + } + } + output.values[row * g.out_width + ox] = finish(sum, oc); + } + }); + return output; +} + +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dRouteCommon.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dRouteCommon.cpp new file mode 100644 index 00000000000..ab9cb74b1c5 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dRouteCommon.cpp @@ -0,0 +1,135 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include + +namespace vkcompute::shader_profiling::convolution { + +void supported(bool condition, const std::string& message) { + if (!condition) { + throw UnsupportedRoute(message); + } +} + +void check_options(const Json& options, bool intermediate_storage) { + if (!options.is_object()) { + throw std::invalid_argument("Route options must be an object"); + } + for (const auto& item : options.items()) { + if (!intermediate_storage || item.key() != "intermediate_storage") { + throw std::invalid_argument( + "Unknown convolution route option: " + item.key()); + } + } +} + +std::array +spatial_parameter(ComputeGraph& graph, ValueRef value, bool positive) { + const std::vector values = *graph.get_int_list(value); + supported(values.size() == 2, "Conv2d requires two spatial parameter values"); + for (int64_t item : values) { + supported( + item >= (positive ? 1 : 0) && item <= INT32_MAX, + "Conv2d spatial parameters are out of range"); + } + return {values[0], values[1]}; +} + +void constant_tensor( + ComputeGraph& graph, + ValueRef value, + const std::string& name) { + supported(graph.val_is_tref(value), name + " requires storage: constant"); +} + +int64_t product(int64_t a, int64_t b) { + supported( + a >= 0 && b >= 0 && + (b == 0 || a <= std::numeric_limits::max() / b), + "Convolution route dimensions overflow int64"); + return a * b; +} + +int64_t padded4(int64_t value) { + supported( + value > 0 && value <= INT32_MAX, + "Convolution channel dimension is out of range"); + return (value + 3) / 4 * 4; +} + +void grouped_packing(const ConvShape& shape) { + if (shape.groups > 1) { + supported( + shape.input[1] / shape.groups % 4 == 0 && + shape.output[1] / shape.groups % 4 == 0, + "Grouped convolution requires input/output channels per group divisible by 4"); + } +} + +Json selected(const std::string& name, const Json& options) { + return {{"name", name}, {"options", options}}; +} + +ConvShape conv_shape_base( + ComputeGraph& graph, + ValueRef input, + ValueRef weight, + ValueRef output, + int64_t groups, + size_t weight_rank) { + supported( + graph.val_is_tensor(input) && graph.val_is_tensor(output), + "Convolution input/output must be GPU tensors"); + constant_tensor(graph, weight, "Convolution weights"); + ConvShape shape{ + graph.sizes_of(input), + graph.sizes_of(output), + graph.sizes_of(weight), + {}, + groups}; + supported( + shape.input.size() == 4 && shape.output.size() == 4 && + shape.weight.size() == weight_rank, + "Convolution route supports rank-4 NCHW input/output and canonical weights"); + for (const auto* sizes : {&shape.input, &shape.output, &shape.weight}) { + for (int64_t size : *sizes) { + supported( + size > 0 && size <= INT32_MAX, + "Convolution tensor dimensions are out of range"); + } + } + supported( + shape.groups > 0 && shape.groups <= INT32_MAX && + shape.input[1] % shape.groups == 0 && + shape.output[1] % shape.groups == 0, + "Convolution channels must be divisible by positive groups"); + supported( + shape.input[0] == shape.output[0] && shape.weight[0] == shape.output[1], + "Convolution input/output/weight dimensions disagree"); + return shape; +} + +void check_conv_output( + ComputeGraph& graph, + const ConvShape& shape, + ValueRef stride_value, + ValueRef padding_value, + ValueRef dilation_value) { + const auto stride = spatial_parameter(graph, stride_value, true); + const auto padding = spatial_parameter(graph, padding_value, false); + const auto dilation = spatial_parameter(graph, dilation_value, true); + for (size_t i = 0; i < 2; ++i) { + const int64_t numerator = shape.input[i + 2] + 2 * padding[i] - + product(dilation[i], shape.kernel[i] - 1) - 1; + supported( + numerator >= 0 && numerator / stride[i] + 1 == shape.output[i + 2], + "Convolution declared output shape disagrees with spatial parameters"); + } +} + +} // namespace vkcompute::shader_profiling::convolution diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dRouteCommon.h b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dRouteCommon.h new file mode 100644 index 00000000000..947372b14db --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dRouteCommon.h @@ -0,0 +1,47 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once +#include + +#include + +namespace vkcompute::shader_profiling::convolution { + +struct ConvShape { + std::vector input; + std::vector output; + std::vector weight; + std::array kernel; + int64_t groups; +}; + +void supported(bool condition, const std::string& message); +void check_options(const Json& options, bool intermediate_storage = false); +std::array +spatial_parameter(ComputeGraph& graph, ValueRef value, bool positive); +void constant_tensor( + ComputeGraph& graph, + ValueRef value, + const std::string& name); +int64_t product(int64_t a, int64_t b); +int64_t padded4(int64_t value); +void grouped_packing(const ConvShape& shape); +Json selected(const std::string& name, const Json& options = Json::object()); +ConvShape conv_shape_base( + ComputeGraph& graph, + ValueRef input, + ValueRef weight, + ValueRef output, + int64_t groups, + size_t weight_rank); +void check_conv_output( + ComputeGraph& graph, + const ConvShape& shape, + ValueRef stride, + ValueRef padding, + ValueRef dilation); +} // namespace vkcompute::shader_profiling::convolution diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dRoutes.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dRoutes.cpp new file mode 100644 index 00000000000..48da7d5d6ce --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dRoutes.cpp @@ -0,0 +1,235 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include +#include +#include +#include + +#include +#include + +namespace vkcompute::shader_profiling { +using namespace convolution; +namespace { + +std::optional storage_option(const Json& options) { + check_options(options, true); + if (!options.contains("intermediate_storage")) { + return std::nullopt; + } + if (!options.at("intermediate_storage").is_string()) { + throw std::invalid_argument( + "intermediate_storage must be a storage type name"); + } + const std::string storage = options.at("intermediate_storage"); + if (storage == "buffer") { + return utils::kBuffer; + } + if (storage == "texture2d") { + return utils::kTexture2D; + } + if (storage == "texture3d") { + return utils::kTexture3D; + } + throw std::invalid_argument("Unknown intermediate_storage: " + storage); +} + +std::string storage_name(utils::StorageType storage) { + switch (storage) { + case utils::kBuffer: + return "buffer"; + case utils::kTexture2D: + return "texture2d"; + case utils::kTexture3D: + return "texture3d"; + } + throw std::invalid_argument("Unknown intermediate tensor storage"); +} + +bool floating(vkapi::ScalarType dtype) { + return dtype == vkapi::kFloat || dtype == vkapi::kHalf; +} + +ConvShape conv_shape(ComputeGraph& graph, const std::vector& args) { + if (args.size() != 10u) + throw std::invalid_argument( + "Convolution route received wrong argument count"); + const ValueRef input = args[0]; + const ValueRef output = args.back(); + const ValueRef weight = args[1]; + ConvShape shape = + conv_shape_base(graph, input, weight, output, graph.get_int(args[8]), 4); + + supported( + !graph.get_bool(args[6]), + "Explicit FP convolution routes support forward convolution only"); + supported( + shape.input[0] == 1, "Explicit FP convolution routes require batch 1"); + supported( + shape.weight[1] == shape.input[1] / shape.groups, + "FP convolution weight channels disagree"); + spatial_parameter(graph, args[7], false); + shape.kernel = {shape.weight[2], shape.weight[3]}; + check_conv_output(graph, shape, args[3], args[4], args[5]); + + const auto weight_dtype = graph.dtype_of(weight); + const auto effective_weight_dtype = weight_dtype == vkapi::kHalf && + !graph.context()->adapter_ptr()->has_full_float16_buffers_support() + ? vkapi::kFloat + : weight_dtype; + supported( + floating(graph.dtype_of(input)) && + graph.dtype_of(input) == graph.dtype_of(output) && + floating(weight_dtype) && + effective_weight_dtype == graph.dtype_of(input), + "FP convolution requires matching effective input/output/weight dtypes"); + supported( + graph.storage_type_of(input) == utils::kTexture3D && + graph.storage_type_of(output) == utils::kTexture3D && + graph.packed_dim_of(input) == WHCN::kChannelsDim && + graph.packed_dim_of(output) == WHCN::kChannelsDim, + "Explicit FP convolution routes require channels-packed texture3d IO"); + if (graph.val_is_not_none(args[2])) { + constant_tensor(graph, args[2], "FP convolution bias"); + supported( + graph.dtype_of(args[2]) == weight_dtype && + graph.sizes_of(args[2]) == std::vector{shape.output[1]}, + "FP convolution bias must match weight dtype and output channels"); + } + return shape; +} + +void check_fp_im2col_buffer_resources( + ComputeGraph& graph, + ValueRef input, + const ConvShape& shape, + std::optional override) { + const auto* adapter = graph.context()->adapter_ptr(); + const int64_t k = product( + product(padded4(shape.input[1]), shape.kernel[0]), shape.kernel[1]); + supported(k <= INT32_MAX, "FP im2col reduction size must fit int32"); + const int64_t k4 = k / 4; + const int64_t max2d = adapter->max_texture2d_dim(); + const int64_t elem_size = vkapi::element_size(graph.dtype_of(input)); + const int64_t bytes_per_row = product(product(shape.output[3], k), elem_size); + constexpr int64_t kScratchBudgetBytes = 16 * 1024 * 1024; + const int64_t rows = std::min( + shape.output[2], + std::max(int64_t(1), kScratchBudgetBytes / bytes_per_row)); + const int64_t matrix_rows = product(rows, shape.output[3]); + const int64_t max3d = adapter->max_texture3d_dim(); + const bool fits2d = k4 <= max2d && matrix_rows <= max2d; + const bool fits3d = shape.output[3] <= max3d && rows <= max3d && k4 <= max3d; + const utils::StorageType storage = override.value_or( + graph.device_is_mali() ? utils::kBuffer + : fits2d ? utils::kTexture2D + : fits3d ? utils::kTexture3D + : utils::kBuffer); + if (storage == utils::kBuffer) { + supported( + product(rows, bytes_per_row) <= graph.max_buffer_numel(), + "FP im2col scratch buffer exceeds device range"); + } +} + +} // namespace + +Json build_conv2d_direct( + ComputeGraph& graph, + const std::vector& args, + const Json& options) { + check_options(options); + const ConvShape shape = conv_shape(graph, args); + grouped_packing(shape); + conv2d_direct_impl( + graph, + args[0], + args[1], + args[2], + args[3], + args[4], + args[5], + args[6], + args[7], + args[8], + args[9], + false, + 0, + 0); + return selected("direct"); +} + +Json build_conv2d_im2col( + ComputeGraph& graph, + const std::vector& args, + const Json& options) { + const auto storage = storage_option(options); + const ConvShape shape = conv_shape(graph, args); + supported(shape.groups == 1, "FP im2col convolution requires groups 1"); + check_fp_im2col_buffer_resources(graph, args[0], shape, storage); + const size_t first = graph.execute_nodes().size(); + conv2d_gemm_impl( + graph, + args[0], + args[1], + args[2], + args[3], + args[4], + args[5], + args[9], + false, + 0, + 0, + storage); + const ValueRef scratch = + graph.execute_nodes().at(first)->args().at(0).refs.at(0); + return selected( + "im2col", + {{"intermediate_storage", storage_name(graph.storage_type_of(scratch))}}); +} + +Json inspect_conv2d_route( + ComputeGraph& graph, + const std::vector&, + size_t first, + size_t end) { + for (size_t i = first; i < end; ++i) { + const auto& node = graph.execute_nodes().at(i); + const std::string name = node->name(); + if (name.rfind("conv2d_im2col_", 0) == 0) { + const ValueRef scratch = node->args().at(0).refs.at(0); + return selected( + "im2col", + {{"intermediate_storage", + storage_name(graph.storage_type_of(scratch))}}); + } + if (name.rfind("conv2d_pw", 0) == 0) { + return selected("pointwise"); + } + if (name.rfind("conv2d_dw", 0) == 0) { + return selected("depthwise"); + } + if (name.rfind("conv_transpose2d", 0) == 0) { + return selected("transposed"); + } + if (name == "conv2d_float" || name == "conv2d_half") { + return selected("direct"); + } + } + return Json(); +} + +REGISTER_OPERATOR_ROUTES { + REGISTER_OPERATOR_ROUTE( + aten.convolution.default, direct, build_conv2d_direct); + REGISTER_OPERATOR_ROUTE( + aten.convolution.default, im2col, build_conv2d_im2col); + REGISTER_ROUTE_INSPECTOR(aten.convolution.default, inspect_conv2d_route); +} + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dRoutes.h b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dRoutes.h new file mode 100644 index 00000000000..0af21aa769d --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dRoutes.h @@ -0,0 +1,26 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include + +namespace vkcompute::shader_profiling { +Json build_conv2d_direct( + ComputeGraph& graph, + const std::vector& args, + const Json& options); +Json build_conv2d_im2col( + ComputeGraph& graph, + const std::vector& args, + const Json& options); +Json inspect_conv2d_route( + ComputeGraph& graph, + const std::vector& args, + size_t first, + size_t end); +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dThroughput.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dThroughput.cpp new file mode 100644 index 00000000000..b52a03eb307 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dThroughput.cpp @@ -0,0 +1,43 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include + +namespace vkcompute::shader_profiling { +namespace { + +double conv2d_flops(const Json& spec) { + const auto weight_sizes = throughput::tensor_argument(spec, 1) + .at("sizes") + .get>(); + if (weight_sizes.size() != 4) + throw std::invalid_argument( + "Conv2d work count requires rank-4 input/output and supported weight shape"); + if (spec.at("args").at(6).at("bool").get()) + throw std::invalid_argument( + "Conv2d work count does not support transposed convolution"); + return throughput::convolution_flops( + spec, + 1, + 2, + weight_sizes, + spec.at("args").at(3).at("int_list").get>(), + spec.at("args").at(4).at("int_list").get>(), + spec.at("args").at(5).at("int_list").get>(), + spec.at("args").at(8).at("int")); +} +} // namespace + +REGISTER_THROUGHPUT_FUNCTIONS { + REGISTER_OPERATOR_MEMORY_TRAFFIC( + aten.convolution.default, throughput::tensor_traffic); + REGISTER_OPERATOR_COMPUTE_FLOPS(aten.convolution.default, conv2d_flops); + REGISTER_SHADER_MEMORY_TRAFFIC(conv2d, throughput::tensor_traffic); +} + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dThroughputCommon.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dThroughputCommon.cpp new file mode 100644 index 00000000000..21dfdc9c837 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dThroughputCommon.cpp @@ -0,0 +1,76 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include + +namespace vkcompute::shader_profiling::throughput { + +double convolution_flops( + const Json& spec, + size_t weight_index, + size_t bias_index, + const std::vector& weight_sizes, + const std::vector& stride, + const std::vector& padding, + const std::vector& dilation, + int64_t groups) { + const Json& input = tensor_argument(spec, 0); + const Json& weight = tensor_argument(spec, weight_index); + const Json& output = output_tensor(spec); + const auto input_sizes = input.at("sizes").get>(); + const auto output_sizes = output.at("sizes").get>(); + tensor_elements(input); + tensor_elements(weight); + const double output_elements = tensor_elements(output); + if (input_sizes.size() != 4 || output_sizes.size() != 4 || + weight_sizes.size() != 4u) { + throw std::invalid_argument( + "Conv2d work count requires rank-4 input/output and supported weight shape"); + } + const std::vector kernel{weight_sizes[2], weight_sizes[3]}; + if (groups <= 0 || input_sizes[1] % groups != 0 || + output_sizes[1] % groups != 0 || stride.size() != 2 || + padding.size() != 2 || dilation.size() != 2 || + input_sizes[0] != output_sizes[0] || weight_sizes[0] != output_sizes[1]) { + throw std::invalid_argument( + "Conv2d work count dimensions/groups/parameters disagree"); + } + for (size_t i = 0; i < 2; ++i) { + if (kernel[i] <= 0 || stride[i] <= 0 || padding[i] < 0 || + dilation[i] <= 0) { + throw std::invalid_argument( + "Conv2d work count requires positive spatial parameters"); + } + const double expected = + std::floor( + (double(input_sizes[i + 2]) + 2 * double(padding[i]) - + double(dilation[i]) * double(kernel[i] - 1) - 1) / + double(stride[i])) + + 1; + if (expected != double(output_sizes[i + 2])) { + throw std::invalid_argument( + "Conv2d work count output shape does not match stride/padding/dilation"); + } + } + const double inner = + double(input_sizes[1] / groups) * double(kernel[0]) * double(kernel[1]); + if (weight_sizes[1] != input_sizes[1] / groups) { + throw std::invalid_argument( + "Conv2d work count weight channels/kernel dimensions disagree"); + } + const bool bias = !spec.at("args").at(bias_index).is_null(); + if (bias && + tensor_elements(tensor_argument(spec, bias_index)) != output_sizes[1]) { + throw std::invalid_argument( + "Conv2d work count bias must match output channels"); + } + return output_elements * (2 * inner + (bias ? 1 : 0)); +} + +} // namespace vkcompute::shader_profiling::throughput diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dThroughputCommon.h b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dThroughputCommon.h new file mode 100644 index 00000000000..382b31e173a --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/Conv2dThroughputCommon.h @@ -0,0 +1,21 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once +#include + +namespace vkcompute::shader_profiling::throughput { + +double convolution_flops( + const Json& spec, + size_t weight_index, + size_t bias_index, + const std::vector& weight_sizes, + const std::vector& stride, + const std::vector& padding, + const std::vector& dilation, + int64_t groups); +} // namespace vkcompute::shader_profiling::throughput diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv1d/Conv1dReference.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv1d/Conv1dReference.cpp new file mode 100644 index 00000000000..d8f47912a56 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv1d/Conv1dReference.cpp @@ -0,0 +1,56 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include +#include +namespace vkcompute::shader_profiling::reference { +Tensor +conv1d_reference(const Json& spec, const TensorMap& tensors, size_t threads) { + const Tensor& input = tensor_arg(spec, 0, tensors); + const Tensor& weight = tensor_arg(spec, 1, tensors); + require( + input.sizes.size() == 3 && weight.sizes.size() == 3, + "Conv1d requires rank3 input and weight"); + const int64_t n = input.sizes[0], ci = input.sizes[1], l = input.sizes[2]; + const int64_t co = weight.sizes[0], cg = weight.sizes[1], k = weight.sizes[2]; + const int64_t groups = spec.at("args").at(8).at("int"); + const int64_t stride = spec.at("args").at(3).at("int_list").at(0); + const int64_t padding = spec.at("args").at(4).at("int_list").at(0); + const int64_t dilation = spec.at("args").at(5).at("int_list").at(0); + require( + groups > 0 && ci == cg * groups && co % groups == 0 && stride > 0 && + dilation > 0, + "Invalid Conv1d dimensions or groups"); + require( + !spec.at("args").at(6).at("bool").get(), + "Transposed Conv1d reference is unsupported"); + const int64_t lo = (l + 2 * padding - dilation * (k - 1) - 1) / stride + 1; + require(lo > 0, "Conv1d output length must be positive"); + const Tensor* bias = + spec.at("args").at(2).is_null() ? nullptr : &tensor_arg(spec, 2, tensors); + Tensor out{{n, co, lo}, std::vector(n * co * lo)}; + parallel_for(n * co, threads, [&](size_t slice) { + const int64_t batch = slice / co, oc = slice % co, + group = oc / (co / groups); + for (int64_t x = 0; x < lo; ++x) { + float sum = 0; + for (int64_t c = 0; c < cg; ++c) + for (int64_t j = 0; j < k; ++j) { + const int64_t ix = x * stride - padding + j * dilation; + if (ix >= 0 && ix < l) + sum = std::fma( + float(input.values[(batch * ci + group * cg + c) * l + ix]), + float(weight.values[(oc * cg + c) * k + j]), + sum); + } + out.values[slice * lo + x] = + sum + (bias ? float(bias->values[oc]) : 0.0f); + } + }); + return out; +} +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv1d/Conv1dReference.h b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv1d/Conv1dReference.h new file mode 100644 index 00000000000..f0432816534 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv1d/Conv1dReference.h @@ -0,0 +1,11 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once +#include +namespace vkcompute::shader_profiling::reference { +Tensor conv1d_reference(const Json&, const TensorMap&, size_t); +} diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv1d/cases/dw_correctness.json b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv1d/cases/dw_correctness.json new file mode 100644 index 00000000000..08eb7a17655 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv1d/cases/dw_correctness.json @@ -0,0 +1,796 @@ +[ + { + "name": "conv1d dw_correctness [1, 16, 64, 3, 1, 1, 1, False]", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [16, 1, 3], + "dtype": "float32", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1] + }, + { + "int_list": [1] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 16 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d dw_correctness [1, 5, 64, 3, 1, 1, 1, False]", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 5, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [5, 1, 3], + "dtype": "float32", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1] + }, + { + "int_list": [1] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 5 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 5, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d dw_correctness [2, 7, 64, 3, 1, 1, 1, False]", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [2, 7, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [7, 1, 3], + "dtype": "float32", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1] + }, + { + "int_list": [1] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 7 + }, + { + "tensor": { + "name": "output", + "sizes": [2, 7, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d dw_correctness [1, 16, 64, 7, 1, 3, 2, False]", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [16, 1, 7], + "dtype": "float32", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1] + }, + { + "int_list": [3] + }, + { + "int_list": [2] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 16 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 58], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d dw_correctness [1, 7, 32, 5, 1, 2, 1, False]", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 7, 32], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [7, 1, 5], + "dtype": "float32", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1] + }, + { + "int_list": [2] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 7 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 7, 32], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d dw_correctness [1, 64, 32, 3, 2, 1, 1, False]", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 32], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 1, 3], + "dtype": "float32", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [2] + }, + { + "int_list": [1] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 64 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 16], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d dw_correctness [1, 32, 128, 5, 1, 2, 1, True]", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 1, 5], + "dtype": "float32", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1] + }, + { + "int_list": [2] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 32 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d dw_correctness [2, 16, 64, 3, 1, 1, 1, True]", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [2, 16, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [16, 1, 3], + "dtype": "float32", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1] + }, + { + "int_list": [1] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 16 + }, + { + "tensor": { + "name": "output", + "sizes": [2, 16, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d dw_correctness [1, 5, 64, 3, 1, 1, 1, True]", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 5, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [5, 1, 3], + "dtype": "float32", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [5], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1] + }, + { + "int_list": [1] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 5 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 5, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d dw_correctness [1, 13, 48, 3, 2, 1, 1, True]", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 13, 48], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [13, 1, 3], + "dtype": "float32", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [13], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [2] + }, + { + "int_list": [1] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 13 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 13, 24], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv1d/cases/dw_performance.json b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv1d/cases/dw_performance.json new file mode 100644 index 00000000000..bd68751ee35 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv1d/cases/dw_performance.json @@ -0,0 +1,474 @@ +[ + { + "name": "conv1d dw_performance [1, 256, 1024, 3, 1, 1, 1, False] float32 height", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 256, 1024], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 1, 3], + "dtype": "float32", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1] + }, + { + "int_list": [1] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 256 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 256, 1024], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d dw_performance [1, 128, 4096, 31, 1, 15, 1, False] float32 height", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128, 4096], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 1, 31], + "dtype": "float32", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1] + }, + { + "int_list": [15] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 128 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 128, 4096], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d dw_performance [1, 256, 1024, 3, 1, 1, 1, False] float16 height", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 256, 1024], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 1, 3], + "dtype": "float16", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1] + }, + { + "int_list": [1] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 256 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 256, 1024], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d dw_performance [1, 128, 4096, 31, 1, 15, 1, False] float16 height", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128, 4096], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 1, 31], + "dtype": "float16", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1] + }, + { + "int_list": [15] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 128 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 128, 4096], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d dw_performance [1, 512, 2048, 5, 1, 2, 1, True] float32 height", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 512, 2048], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [512, 1, 5], + "dtype": "float32", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [512], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1] + }, + { + "int_list": [2] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 512 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 512, 2048], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d dw_performance [1, 512, 2048, 5, 1, 2, 1, True] float16 height", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 512, 2048], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [512, 1, 5], + "dtype": "float16", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [512], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1] + }, + { + "int_list": [2] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 512 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 512, 2048], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv1d/cases/pw_correctness.json b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv1d/cases/pw_correctness.json new file mode 100644 index 00000000000..c1fee9a524d --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv1d/cases/pw_correctness.json @@ -0,0 +1,1129 @@ +[ + { + "name": "conv1d pw_correctness [1, 16, 32, 64, False]", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 16, 1], + "dtype": "float32", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1] + }, + { + "int_list": [0] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d pw_correctness [1, 32, 16, 128, False]", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [16, 32, 1], + "dtype": "float32", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1] + }, + { + "int_list": [0] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d pw_correctness [1, 64, 64, 32, False]", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 32], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 64, 1], + "dtype": "float32", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1] + }, + { + "int_list": [0] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 32], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d pw_correctness [2, 16, 32, 64, False]", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [2, 16, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 16, 1], + "dtype": "float32", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1] + }, + { + "int_list": [0] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [2, 32, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d pw_correctness [1, 5, 7, 64, False]", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 5, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [7, 5, 1], + "dtype": "float32", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1] + }, + { + "int_list": [0] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 7, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d pw_correctness [1, 13, 17, 48, False]", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 13, 48], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [17, 13, 1], + "dtype": "float32", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1] + }, + { + "int_list": [0] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 17, 48], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d pw_correctness [1, 7, 5, 32, False]", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 7, 32], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [5, 7, 1], + "dtype": "float32", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1] + }, + { + "int_list": [0] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 5, 32], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d pw_correctness [1, 16, 32, 64, True]", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 16, 1], + "dtype": "float32", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1] + }, + { + "int_list": [0] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d pw_correctness [1, 32, 16, 128, True]", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [16, 32, 1], + "dtype": "float32", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1] + }, + { + "int_list": [0] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d pw_correctness [1, 128, 256, 16, True]", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128, 16], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 128, 1], + "dtype": "float32", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [256], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1] + }, + { + "int_list": [0] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 256, 16], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d pw_correctness [2, 16, 32, 64, True]", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [2, 16, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 16, 1], + "dtype": "float32", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1] + }, + { + "int_list": [0] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [2, 32, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d pw_correctness [1, 5, 7, 64, True]", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 5, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [7, 5, 1], + "dtype": "float32", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [7], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1] + }, + { + "int_list": [0] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 7, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d pw_correctness [1, 13, 17, 48, True]", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 13, 48], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [17, 13, 1], + "dtype": "float32", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [17], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1] + }, + { + "int_list": [0] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 17, 48], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d pw_correctness [2, 5, 13, 64, True]", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [2, 5, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [13, 5, 1], + "dtype": "float32", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [13], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1] + }, + { + "int_list": [0] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [2, 13, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv1d/cases/pw_performance.json b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv1d/cases/pw_performance.json new file mode 100644 index 00000000000..1ad860ca7bc --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv1d/cases/pw_performance.json @@ -0,0 +1,646 @@ +[ + { + "name": "conv1d pw_performance [1, 256, 512, 1024, False] float32 height", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 256, 1024], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [512, 256, 1], + "dtype": "float32", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1] + }, + { + "int_list": [0] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 512, 1024], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d pw_performance [1, 512, 256, 2048, False] float32 height", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 512, 2048], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 512, 1], + "dtype": "float32", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1] + }, + { + "int_list": [0] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 256, 2048], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d pw_performance [1, 256, 512, 1024, False] float16 height", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 256, 1024], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [512, 256, 1], + "dtype": "float16", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1] + }, + { + "int_list": [0] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 512, 1024], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d pw_performance [1, 512, 256, 2048, False] float16 height", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 512, 2048], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 512, 1], + "dtype": "float16", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1] + }, + { + "int_list": [0] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 256, 2048], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d pw_performance [1, 256, 512, 1024, True] float32 height", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 256, 1024], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [512, 256, 1], + "dtype": "float32", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [512], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1] + }, + { + "int_list": [0] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 512, 1024], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d pw_performance [1, 128, 128, 4096, True] float32 height", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128, 4096], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 128, 1], + "dtype": "float32", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1] + }, + { + "int_list": [0] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 128, 4096], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d pw_performance [1, 256, 512, 1024, True] float16 height", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 256, 1024], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [512, 256, 1], + "dtype": "float16", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [512], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1] + }, + { + "int_list": [0] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 512, 1024], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv1d pw_performance [1, 128, 128, 4096, True] float16 height", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128, 4096], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 128, 1], + "dtype": "float16", + "storage": "constant", + "layout": "height", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1] + }, + { + "int_list": [0] + }, + { + "int_list": [1] + }, + { + "bool": false + }, + { + "int_list": [0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 128, 4096], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "height", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv2d/DepthwiseRoutes.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv2d/DepthwiseRoutes.cpp new file mode 100644 index 00000000000..bcf6b02e352 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv2d/DepthwiseRoutes.cpp @@ -0,0 +1,300 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace vkcompute { + +struct Conv2dDWParams final { + utils::ivec2 overlay_region; + int in_group_size; +}; + +struct OutputParams final { + float out_min; + float out_max; +}; + +static std::string pick_conv2d_dw_shader_with_selector( + ComputeGraph& graph, + const ValueRef weight_data, + const ValueRef out, + const bool stride_equals_dilation, + const bool clamp_out, + const std::string& impl_selector) { + std::string kernel_name = "conv2d_dw"; + kernel_name.reserve(40); + + const auto& weight_sizes = graph.get_tref(weight_data)->sizes; + const bool is_3x3 = weight_sizes.at(2) == 3 && weight_sizes.at(3) == 3; + const bool is_5x5 = weight_sizes.at(2) == 5 && weight_sizes.at(3) == 5; + + if (!stride_equals_dilation && (is_3x3 || is_5x5)) { + kernel_name += "_sned"; + } + + if (is_3x3) { + kernel_name += "_output_tile_3x3"; + if (impl_selector == "b1x1") { + if (stride_equals_dilation) { + kernel_name += "_b1x1"; + } + } else if (impl_selector == "b4x2") { + } else { + if (stride_equals_dilation && graph.device_is_mali()) { + kernel_name += "_b1x1"; + } + } + } else if (is_5x5) { + kernel_name += "_output_tile_5x5"; + } + + if (clamp_out) { + kernel_name += "_clamp"; + } + add_dtype_suffix(kernel_name, graph.dtype_of(out)); + + return kernel_name; +} + +static GlobalWorkGrid conv2d_dw_gwg_fn( + ComputeGraph* graph, + const vkapi::ShaderInfo& shader, + const std::vector& args, + const std::vector& resize_args) { + (void)resize_args; + const ValueRef out = args.at(0).refs.at(0); + + const bool uses_output_tile = + shader.kernel_name.find("_output_tile") != std::string::npos; + + if (uses_output_tile) { + const bool is_sned = shader.kernel_name.find("_sned") != std::string::npos; + const utils::uvec3 image_extents = graph->create_gwg(out).extents(); + + if (is_sned) { + return GlobalWorkGrid( + {image_extents[0] * image_extents[1], image_extents[2], 1u}, + kTiledWorkGrid); + } + + uint32_t batch_x = 4u; + uint32_t batch_y = 2u; + if (shader.kernel_name.find("_b1x1") != std::string::npos) { + batch_x = 1u; + batch_y = 1u; + } + + uint32_t scaled_x = utils::div_up(image_extents[0], batch_x); + uint32_t scaled_y = utils::div_up(image_extents[1], batch_y); + return GlobalWorkGrid( + {scaled_x * scaled_y, image_extents[2], 1u}, kTiledWorkGrid); + } + + const utils::uvec3 base_extents = graph->create_gwg(out).extents(); + return graph->create_linear_gwg( + base_extents[0] * base_extents[1] * base_extents[2]); +} + +static LocalWorkGroup conv2d_dw_lwg_fn( + ComputeGraph* graph, + const vkapi::ShaderInfo& shader, + const GlobalWorkGrid& gwg, + const std::vector& args, + const std::vector& resize_args) { + (void)graph; + (void)shader; + (void)gwg; + (void)args; + (void)resize_args; + return LocalWorkGroup(64u, 1u, 1u); +} + +static ValueRef prepack_dw_weights(ComputeGraph& graph, const ValueRef vref) { + const auto original_sizes = graph.sizes_of(vref); + + int64_t out_channels_padded = + utils::align_up_4(utils::val_at(-4, original_sizes)); + int64_t height = utils::val_at(-2, original_sizes); + int64_t width = utils::val_at(-1, original_sizes); + + const std::vector final_sizes = { + 4, out_channels_padded / 4, height * width}; + + ValueRef v = graph.add_tensor( + final_sizes, + graph.dtype_of(vref), + utils::kTexture2D, + utils::kChannelsPacked); + + std::string kernel_name = "conv2d_dw_prepack_weights"; + add_dtype_suffix(kernel_name, graph.dtype_of(v)); + add_dtype_suffix(kernel_name, graph.get_staging_dtype_for(vref)); + + const auto original_sizes_pc = utils::make_ivec4(original_sizes, true); + graph.prepack_nodes().emplace_back(new PrepackNode( + graph, + VK_KERNEL_FROM_STR(kernel_name), + graph.create_gwg(v), + graph.create_lwg(v), + vref, + v, + {}, + {graph.packed_dim_of(v)}, + {graph.sizes_pc_of(v), + PushConstantDataInfo(&original_sizes_pc, sizeof(original_sizes_pc))})); + + return v; +} + +static void conv2d_dw_with_selector( + ComputeGraph& graph, + const ValueRef in, + const ValueRef weight_data, + const ValueRef bias, + const ValueRef stride, + const ValueRef padding, + const ValueRef dilation, + const ValueRef out, + const std::string& impl_selector) { + ValueRef arg_weight = prepack_dw_weights(graph, weight_data); + ValueRef arg_bias = prepack_biases( + graph, bias, weight_data, false, utils::kTexture2D, utils::kWidthPacked); + + check_conv_args(graph, in, out); + + Kernel2dParams kernel_params = create_kernel2d_params( + graph, weight_data, false, stride, padding, dilation); + + const bool stride_equals_dilation = + (kernel_params.stride[0] == kernel_params.dilation[0] && + kernel_params.stride[1] == kernel_params.dilation[1]); + + const auto& overlay_region = utils::make_ivec2({ + kernel_params.kernel_size[0] + + (kernel_params.kernel_size[0] - 1) * (kernel_params.dilation[0] - 1), + kernel_params.kernel_size[1] + + (kernel_params.kernel_size[1] - 1) * (kernel_params.dilation[1] - 1), + }); + const auto weight_sizes = graph.sizes_of(weight_data); + const int32_t in_group_size = + utils::safe_downcast(utils::align_up_4(weight_sizes.at(1))); + Conv2dDWParams extra_params = {overlay_region, in_group_size}; + + OutputParams out_params = { + std::numeric_limits::lowest(), std::numeric_limits::max()}; + + std::string kernel_name = pick_conv2d_dw_shader_with_selector( + graph, weight_data, out, stride_equals_dilation, false, impl_selector); + + vkapi::ShaderInfo shader = VK_KERNEL_FROM_STR(kernel_name); + + vkapi::ParamsBindList param_buffers; + std::vector push_constants; + + const bool uses_output_tile = + kernel_name.find("_output_tile") != std::string::npos; + + if (uses_output_tile) { + const utils::ivec4 kernel_param_size_stride = { + kernel_params.kernel_size[0], + kernel_params.kernel_size[1], + kernel_params.stride[0], + kernel_params.stride[1]}; + + const utils::ivec4 kernel_param_pad_dial = { + kernel_params.padding[0], + kernel_params.padding[1], + kernel_params.dilation[0], + kernel_params.dilation[1]}; + + push_constants = { + graph.logical_limits_pc_of(out), + graph.sizes_pc_of(in), + PushConstantDataInfo( + &kernel_param_size_stride, sizeof(kernel_param_size_stride)), + PushConstantDataInfo( + &kernel_param_pad_dial, sizeof(kernel_param_pad_dial)), + PushConstantDataInfo( + &extra_params, sizeof(extra_params), sizeof(utils::ivec4)), + PushConstantDataInfo(&out_params, sizeof(out_params)), + }; + } else { + param_buffers = { + graph.logical_limits_ubo(out), + graph.sizes_ubo(in), + graph.create_params_buffer(kernel_params), + graph.create_params_buffer(extra_params), + graph.create_params_buffer(out_params), + }; + } + + ValueRef transposed_ref = graph.add_scalar(false); + ValueRef output_padding = graph.add_none(); + + graph.execute_nodes().emplace_back(new DynamicDispatchNode( + graph, + shader, + conv2d_dw_gwg_fn, + conv2d_dw_lwg_fn, + {{out, vkapi::kWrite}, {{in, arg_weight, arg_bias}, vkapi::kRead}}, + param_buffers, + push_constants, + {}, + {weight_data, stride, padding, dilation, transposed_ref, output_padding}, + resize_conv2d_node)); +} + +namespace shader_profiling { +namespace { +template +Json build_dw_route( + ComputeGraph& graph, + const std::vector& args, + const Json& options) { + if (!options.empty()) + throw std::invalid_argument("Depthwise tile routes accept no options"); + if (args.size() != 10 || graph.extract_scalar(args[6]) || + graph.extract_scalar(args[8]) != graph.sizes_of(args[0]).at(1)) + throw UnsupportedRoute( + "Depthwise tile routes require forward depthwise convolution"); + if (!graph.is_texture_storage(args[0]) || + !graph.is_texture_storage(args.back())) + throw UnsupportedRoute("Depthwise tile routes require texture IO"); + conv2d_dw_with_selector( + graph, + args[0], + args[1], + args[2], + args[3], + args[4], + args[5], + args[9], + B1 ? "b1x1" : "b4x2"); + const auto& sizes = graph.sizes_of(args[1]); + const auto stride = *graph.get_int_list(args[3]); + const auto dilation = *graph.get_int_list(args[5]); + const bool selected = + B1 && sizes[2] == 3 && sizes[3] == 3 && stride == dilation; + return {{"name", selected ? "b1x1" : "b4x2"}, {"options", options}}; +} +REGISTER_OPERATOR_ROUTES { + REGISTER_OPERATOR_ROUTE( + aten.convolution.default, b1x1, (build_dw_route)); + REGISTER_OPERATOR_ROUTE( + aten.convolution.default, b4x2, (build_dw_route)); +} +} // namespace +} // namespace shader_profiling +} // namespace vkcompute diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv2d/cases/correctness.json b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv2d/cases/correctness.json new file mode 100644 index 00000000000..d131e8f7a51 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv2d/cases/correctness.json @@ -0,0 +1,1464 @@ +[ + { + "name": "conv2d correctness [[1, 8, 8, 8], 8, [3, 3], [1, 1], [1, 1], [1, 1], False] float32", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 8, 8, 8], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [8, 8, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 8, 8, 8], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d correctness [[1, 16, 8, 8], 8, [3, 3], [1, 1], [1, 1], [1, 1], False] float32", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 8, 8], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [8, 16, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 8, 8, 8], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d correctness [[1, 11, 8, 8], 13, [3, 3], [1, 1], [1, 1], [1, 1], False] float32", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 11, 8, 8], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [13, 11, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 13, 8, 8], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d correctness [[1, 8, 16, 16], 16, [3, 3], [2, 2], [1, 1], [1, 1], False] float32", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 8, 16, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [16, 8, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 8, 8], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d correctness [[1, 3, 16, 16], 8, [3, 3], [2, 2], [1, 1], [1, 1], False] float32", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 3, 16, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [8, 3, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 8, 8, 8], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d correctness [[1, 8, 8, 8], 8, [3, 3], [1, 1], [1, 1], [1, 1], False] float16", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 8, 8, 8], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [8, 8, 3, 3], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 8, 8, 8], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d correctness [[1, 16, 8, 8], 8, [3, 3], [1, 1], [1, 1], [1, 1], False] float16", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 8, 8], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [8, 16, 3, 3], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 8, 8, 8], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d correctness [[1, 11, 8, 8], 13, [3, 3], [1, 1], [1, 1], [1, 1], False] float16", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 11, 8, 8], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [13, 11, 3, 3], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 13, 8, 8], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d correctness [[1, 8, 16, 16], 16, [3, 3], [2, 2], [1, 1], [1, 1], False] float16", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 8, 16, 16], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [16, 8, 3, 3], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 8, 8], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d correctness [[1, 3, 16, 16], 8, [3, 3], [2, 2], [1, 1], [1, 1], False] float16", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 3, 16, 16], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [8, 3, 3, 3], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 8, 8, 8], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d correctness [[1, 8, 8, 8], 8, [3, 3], [1, 1], [1, 1], [1, 1], True] float32", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 8, 8, 8], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [8, 8, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [8], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 8, 8, 8], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d correctness [[1, 16, 16, 16], 16, [3, 3], [1, 1], [1, 1], [1, 1], True] float32", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 16, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [16, 16, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 16, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d correctness [[1, 8, 8, 8], 8, [3, 3], [1, 1], [1, 1], [1, 1], True] float16", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 8, 8, 8], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [8, 8, 3, 3], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [8], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 8, 8, 8], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d correctness [[1, 16, 16, 16], 16, [3, 3], [1, 1], [1, 1], [1, 1], True] float16", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 16, 16], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [16, 16, 3, 3], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [16], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 16, 16], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d correctness [[1, 16, float32 texture3d channels", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": { + "name": "im2col", + "options": { + "intermediate_storage": "v_OPTIONS_INTERMEDIATE_STORAGE" + } + }, + "sweep_variables": { + "v_OPTIONS_INTERMEDIATE_STORAGE": ["buffer", "texture2d", "texture3d"], + "v_SIZES": [ + [1, 16, 16, 16], + [1, 16, 48, 48] + ] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": "v_SIZES", + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [16, 16, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": "v_SIZES", + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d correctness [[1, 16, float16 texture3d channels", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": { + "name": "im2col", + "options": { + "intermediate_storage": "v_OPTIONS_INTERMEDIATE_STORAGE" + } + }, + "sweep_variables": { + "v_OPTIONS_INTERMEDIATE_STORAGE": ["buffer", "texture2d", "texture3d"], + "v_SIZES": [ + [1, 16, 16, 16], + [1, 16, 48, 48] + ] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": "v_SIZES", + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [16, 16, 3, 3], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [16], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": "v_SIZES", + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d correctness [[1, 11, 12, 12], 13, [3, 3], [1, 1], [1, 1], [1, 1], False] float32 {'name': 'im2col', 'options': {'intermediate_storage':", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": { + "name": "im2col", + "options": { + "intermediate_storage": "v_OPTIONS_INTERMEDIATE_STORAGE" + } + }, + "sweep_variables": { + "v_OPTIONS_INTERMEDIATE_STORAGE": ["buffer", "texture2d", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 11, 12, 12], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [13, 11, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 13, 12, 12], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d correctness [[1, 11, 12, 12], 13, [3, 3], [1, 1], [1, 1], [1, 1], False] float16 {'name': 'im2col', 'options': {'intermediate_storage':", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": { + "name": "im2col", + "options": { + "intermediate_storage": "v_OPTIONS_INTERMEDIATE_STORAGE" + } + }, + "sweep_variables": { + "v_OPTIONS_INTERMEDIATE_STORAGE": ["buffer", "texture2d", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 11, 12, 12], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [13, 11, 3, 3], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 13, 12, 12], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv2d/cases/dw_correctness.json b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv2d/cases/dw_correctness.json new file mode 100644 index 00000000000..52abbb085d9 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv2d/cases/dw_correctness.json @@ -0,0 +1,698 @@ +[ + { + "name": "conv2d dw_correctness [[1, 8, 16, 16], [3, 3], [1, 1], [1, 1], [1, 1], False] float32 auto", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 8, 16, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [8, 1, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 8 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 8, 16, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d dw_correctness [[1, 11, 16, 16], [3, 3], [1, 1], [1, 1], [1, 1], False] float32 auto", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 11, 16, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [11, 1, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 11 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 11, 16, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d dw_correctness [[1, 3, 16, 16], [3, 3], [1, 1], [1, 1], [1, 1], False] float32 auto", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 3, 16, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [3, 1, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 3 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 3, 16, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d dw_correctness [[1, 8, 16, 16], [5, 5], [1, 1], [2, 2], [1, 1], False] float32 auto", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 8, 16, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [8, 1, 5, 5], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 8 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 8, 16, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d dw_correctness [[1, 8, 16, 16],", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_SIZES": [ + [8, 1, 3, 3], + [8, 1, 4, 4] + ] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 8, 16, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": "v_WEIGHT_SIZES", + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 8 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 8, 8, 8], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d dw_correctness [[1, 8, 32, 32], [6, 6], [3, 3], [2, 2], [3, 3], False] float32 auto", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 8, 32, 32], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [8, 1, 6, 6], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [3, 3] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 8 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 8, 7, 7], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d dw_correctness [[1, 8, 16, 16], [3, 3], [1, 1], [1, 1], [1, 1], True] float32 auto", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 8, 16, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [8, 1, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [8], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 8 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 8, 16, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d dw_correctness [[1, 8, 32, 32], [7, 7], [1, 1], [6, 6], [2, 2], True] float32 auto", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 8, 32, 32], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [8, 1, 7, 7], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [8], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [6, 6] + }, + { + "int_list": [2, 2] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 8 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 8, 32, 32], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d dw_correctness [[1, 8, 31, 37], [3, 5], [2, 1], [1, 2], [1, 1], True] float32 auto", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 8, 31, 37], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [8, 1, 3, 5], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [8], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [2, 1] + }, + { + "int_list": [1, 2] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 8 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 8, 16, 37], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv2d/cases/dw_performance.json b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv2d/cases/dw_performance.json new file mode 100644 index 00000000000..eaf0dcea897 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv2d/cases/dw_performance.json @@ -0,0 +1,1826 @@ +[ + { + "name": "conv2d dw_performance [[1, 24, 512, 512], [3, 3], [2, 2], [1, 1], [1, 1], False] float32", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "b4x2", "b1x1"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 24, 512, 512], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [24, 1, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 24 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 24, 256, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d dw_performance [[1, 48, 256, 256], [3, 3], [2, 2], [1, 1], [1, 1], False] float32", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "b4x2", "b1x1"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 48, 256, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [48, 1, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 48 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 48, 128, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d dw_performance [[1, 96, 128, 128], [3, 3], [2, 2], [1, 1], [1, 1], False] float32", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "b4x2", "b1x1"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 96, 128, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [96, 1, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 96 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 96, 64, 64], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d dw_performance [[1, 192, 64, 64], [3, 3], [2, 2], [1, 1], [1, 1], False] float32", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "b4x2", "b1x1"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 192, 64, 64], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [192, 1, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 192 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 192, 32, 32], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d dw_performance [[1, 48, 256, 256], [3, 3], [1, 1], [1, 1], [1, 1], False] float32", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "b4x2", "b1x1"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 48, 256, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [48, 1, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 48 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 48, 256, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d dw_performance [[1, 96, 128, 128], [3, 3], [1, 1], [1, 1], [1, 1], False] float32", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "b4x2", "b1x1"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 96, 128, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [96, 1, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 96 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 96, 128, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d dw_performance [[1, 192, 64, 64], [3, 3], [1, 1], [1, 1], [1, 1], False] float32", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "b4x2", "b1x1"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 192, 64, 64], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [192, 1, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 192 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 192, 64, 64], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d dw_performance [[1, 384, 32, 32], [3, 3], [1, 1], [1, 1], [1, 1], False] float32", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "b4x2", "b1x1"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 384, 32, 32], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [384, 1, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 384 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 384, 32, 32], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d dw_performance [[1, 256, 256, 256], [3, 3], [1, 1], [1, 1], [1, 1], False] float32", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "b4x2", "b1x1"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 256, 256, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 1, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 256 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 256, 256, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d dw_performance [[1, 256, 128, 128], [3, 3], [1, 1], [1, 1], [1, 1], False] float32", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "b4x2", "b1x1"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 256, 128, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 1, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 256 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 256, 128, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d dw_performance [[1, 48, 256, 256], [5, 5], [1, 1], [2, 2], [1, 1], False] float32", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "b4x2", "b1x1"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 48, 256, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [48, 1, 5, 5], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 48 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 48, 256, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d dw_performance [[1, 96, 128, 128], [5, 5], [1, 1], [2, 2], [1, 1], False] float32", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "b4x2", "b1x1"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 96, 128, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [96, 1, 5, 5], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 96 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 96, 128, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d dw_performance [[1, 24, 512, 512], [3, 3], [2, 2], [1, 1], [1, 1], False] float16", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "b4x2", "b1x1"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 24, 512, 512], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [24, 1, 3, 3], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 24 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 24, 256, 256], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d dw_performance [[1, 48, 256, 256], [3, 3], [2, 2], [1, 1], [1, 1], False] float16", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "b4x2", "b1x1"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 48, 256, 256], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [48, 1, 3, 3], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 48 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 48, 128, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d dw_performance [[1, 96, 128, 128], [3, 3], [2, 2], [1, 1], [1, 1], False] float16", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "b4x2", "b1x1"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 96, 128, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [96, 1, 3, 3], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 96 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 96, 64, 64], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d dw_performance [[1, 192, 64, 64], [3, 3], [2, 2], [1, 1], [1, 1], False] float16", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "b4x2", "b1x1"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 192, 64, 64], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [192, 1, 3, 3], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 192 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 192, 32, 32], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d dw_performance [[1, 48, 256, 256], [3, 3], [1, 1], [1, 1], [1, 1], False] float16", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "b4x2", "b1x1"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 48, 256, 256], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [48, 1, 3, 3], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 48 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 48, 256, 256], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d dw_performance [[1, 96, 128, 128], [3, 3], [1, 1], [1, 1], [1, 1], False] float16", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "b4x2", "b1x1"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 96, 128, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [96, 1, 3, 3], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 96 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 96, 128, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d dw_performance [[1, 192, 64, 64], [3, 3], [1, 1], [1, 1], [1, 1], False] float16", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "b4x2", "b1x1"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 192, 64, 64], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [192, 1, 3, 3], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 192 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 192, 64, 64], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d dw_performance [[1, 384, 32, 32], [3, 3], [1, 1], [1, 1], [1, 1], False] float16", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "b4x2", "b1x1"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 384, 32, 32], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [384, 1, 3, 3], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 384 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 384, 32, 32], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d dw_performance [[1, 256, 256, 256], [3, 3], [1, 1], [1, 1], [1, 1], False] float16", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "b4x2", "b1x1"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 256, 256, 256], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 1, 3, 3], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 256 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 256, 256, 256], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d dw_performance [[1, 256, 128, 128], [3, 3], [1, 1], [1, 1], [1, 1], False] float16", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "b4x2", "b1x1"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 256, 128, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 1, 3, 3], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 256 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 256, 128, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d dw_performance [[1, 48, 256, 256], [5, 5], [1, 1], [2, 2], [1, 1], False] float16", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "b4x2", "b1x1"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 48, 256, 256], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [48, 1, 5, 5], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 48 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 48, 256, 256], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d dw_performance [[1, 96, 128, 128], [5, 5], [1, 1], [2, 2], [1, 1], False] float16", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "b4x2", "b1x1"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 96, 128, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [96, 1, 5, 5], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 96 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 96, 128, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv2d/cases/performance.json b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv2d/cases/performance.json new file mode 100644 index 00000000000..98accd0c5d9 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv2d/cases/performance.json @@ -0,0 +1,1786 @@ +[ + { + "name": "conv2d performance [[1, 16, 8, 8], 64, [3, 3], [1, 1], [1, 1], [1, 1], False] float32", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 8, 8], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 16, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 8, 8], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d performance [[1, 16, 8, 8], 128, [3, 3], [1, 1], [1, 1], [1, 1], False] float32", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 8, 8], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 16, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 128, 8, 8], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d performance [[1, 128, 18, 24], 64, [3, 3], [1, 1], [1, 1], [1, 1], False] float32", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128, 18, 24], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 128, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 18, 24], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d performance [[1, 64, 36, 48], 32, [3, 3], [1, 1], [1, 1], [1, 1], False] float32", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 36, 48], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 64, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 36, 48], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d performance [[1, 32, 72, 96], 128, [3, 3], [2, 2], [1, 1], [1, 1], False] float32", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 72, 96], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 32, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 128, 36, 48], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d performance [[1, 32, 72, 96], 32, [3, 3], [2, 2], [1, 1], [1, 1], False] float32", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 72, 96], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 32, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 36, 48], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d performance [[1, 64, 36, 48], 64, [3, 3], [2, 2], [1, 1], [1, 1], False] float32", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 36, 48], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 64, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 18, 24], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d performance [[1, 3, 144, 192], 32, [3, 3], [2, 2], [1, 1], [1, 1], False] float32", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 3, 144, 192], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 3, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 72, 96], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d performance [[1, 32, 72, 96], 128, [3, 3], [2, 2], [1, 1], [1, 1], False] float16", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 72, 96], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 32, 3, 3], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 128, 36, 48], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d performance [[1, 32, 72, 96], 32, [3, 3], [2, 2], [1, 1], [1, 1], False] float16", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 72, 96], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 32, 3, 3], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 36, 48], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d performance [[1, 64, 36, 48], 64, [3, 3], [2, 2], [1, 1], [1, 1], False] float16", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 36, 48], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 64, 3, 3], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 18, 24], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d performance [[1, 3, 144, 192], 32, [3, 3], [2, 2], [1, 1], [1, 1], False] float16", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 3, 144, 192], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 3, 3, 3], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 72, 96], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d performance [[1, 128, 18, 24], 64, [3, 3], [1, 1], [1, 1], [1, 1], False] float16", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128, 18, 24], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 128, 3, 3], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 18, 24], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d performance [[1, 64, 36, 48], 32, [3, 3], [1, 1], [1, 1], [1, 1], False] float16", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 36, 48], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 64, 3, 3], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 36, 48], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d performance [[1, 128, float32 texture3d channels", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"], + "v_SIZES": [ + [1, 128, 36, 48], + [1, 128, 18, 24] + ] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": "v_SIZES", + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 128, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": "v_SIZES", + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d performance [[1, 256, 18, 24], 256, [3, 3], [1, 1], [1, 1], [1, 1], True] float32", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 256, 18, 24], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 256, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [256], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 256, 18, 24], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d performance [[1, 64, float32 texture3d channels", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"], + "v_SIZES": [ + [1, 64, 72, 96], + [1, 64, 36, 48] + ] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": "v_SIZES", + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 64, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": "v_SIZES", + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d performance [[1, 32, float32 texture3d channels", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"], + "v_SIZES": [ + [1, 32, 144, 192], + [1, 32, 72, 96] + ] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": "v_SIZES", + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 32, 3, 3], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": "v_SIZES", + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d performance [[1, 128, float16 texture3d channels", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"], + "v_SIZES": [ + [1, 128, 36, 48], + [1, 128, 18, 24] + ] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": "v_SIZES", + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 128, 3, 3], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": "v_SIZES", + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d performance [[1, 256, 18, 24], 256, [3, 3], [1, 1], [1, 1], [1, 1], True] float16", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 256, 18, 24], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 256, 3, 3], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [256], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 256, 18, 24], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d performance [[1, 64, float16 texture3d channels", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"], + "v_SIZES": [ + [1, 64, 72, 96], + [1, 64, 36, 48] + ] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": "v_SIZES", + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 64, 3, 3], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": "v_SIZES", + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d performance [[1, 32, float16 texture3d channels", + "operator": "aten.convolution.default", + "target_ms": 100, + "warmup_runs": 5, + "benchmark_runs": 20, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col", "direct"], + "v_SIZES": [ + [1, 32, 144, 192], + [1, 32, 72, 96] + ] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": "v_SIZES", + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 32, 3, 3], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": "v_SIZES", + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv2d/cases/pw_correctness.json b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv2d/cases/pw_correctness.json new file mode 100644 index 00000000000..a6c2ea90a98 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv2d/cases/pw_correctness.json @@ -0,0 +1,378 @@ +[ + { + "name": "conv2d pw_correctness [1, 16, 32, 8, 8, False] float32 auto", + "operator": "aten.convolution.default", + "route": "auto", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 8, 8], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 16, 1, 1], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 8, 8], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d pw_correctness [1, 32, 16, 8, 8, False] float32 auto", + "operator": "aten.convolution.default", + "route": "auto", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 8, 8], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [16, 32, 1, 1], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 8, 8], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d pw_correctness [1, 16, 32, 8, 8, True] float32 auto", + "operator": "aten.convolution.default", + "route": "auto", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 8, 8], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 16, 1, 1], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 8, 8], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d pw_correctness [1, 13, 27, 8, 8, False] float32 auto", + "operator": "aten.convolution.default", + "route": "auto", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 13, 8, 8], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [27, 13, 1, 1], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 27, 8, 8], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d pw_correctness [1, 33, 17, 8, 8, False] float32 auto", + "operator": "aten.convolution.default", + "route": "auto", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 33, 8, 8], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [17, 33, 1, 1], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 17, 8, 8], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv2d/cases/pw_performance.json b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv2d/cases/pw_performance.json new file mode 100644 index 00000000000..0ab8e008cc4 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/conv2d/cases/pw_performance.json @@ -0,0 +1,1900 @@ +[ + { + "name": "conv2d pw_performance [1, 48, 96, 16, 16, False] float32 auto", + "operator": "aten.convolution.default", + "route": "auto", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 48, 16, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [96, 48, 1, 1], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 96, 16, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d pw_performance [1, 96, 48, 16, 16, False] float32 auto", + "operator": "aten.convolution.default", + "route": "auto", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 96, 16, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [48, 96, 1, 1], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 48, 16, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d pw_performance [1, 48, 96, 256, 256, False] float32 auto", + "operator": "aten.convolution.default", + "route": "auto", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 48, 256, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [96, 48, 1, 1], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 96, 256, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d pw_performance [1, 48, 96, 256, 256, False] float16 auto", + "operator": "aten.convolution.default", + "route": "auto", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 48, 256, 256], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [96, 48, 1, 1], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 96, 256, 256], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d pw_performance [1, 96, 48, 256, 256, False] float32 auto", + "operator": "aten.convolution.default", + "route": "auto", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 96, 256, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [48, 96, 1, 1], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 48, 256, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d pw_performance [1, 96, 48, 256, 256, False] float16 auto", + "operator": "aten.convolution.default", + "route": "auto", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 96, 256, 256], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [48, 96, 1, 1], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 48, 256, 256], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d pw_performance [1, 96, 192, 128, 128, False] float32 auto", + "operator": "aten.convolution.default", + "route": "auto", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 96, 128, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [192, 96, 1, 1], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 192, 128, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d pw_performance [1, 96, 192, 128, 128, False] float16 auto", + "operator": "aten.convolution.default", + "route": "auto", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 96, 128, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [192, 96, 1, 1], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 192, 128, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d pw_performance [1, 192, 96, 128, 128, False] float32 auto", + "operator": "aten.convolution.default", + "route": "auto", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 192, 128, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [96, 192, 1, 1], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 96, 128, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d pw_performance [1, 192, 96, 128, 128, False] float16 auto", + "operator": "aten.convolution.default", + "route": "auto", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 192, 128, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [96, 192, 1, 1], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 96, 128, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d pw_performance [1, 192, 384, 64, 64, False] float32 auto", + "operator": "aten.convolution.default", + "route": "auto", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 192, 64, 64], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [384, 192, 1, 1], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 384, 64, 64], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d pw_performance [1, 192, 384, 64, 64, False] float16 auto", + "operator": "aten.convolution.default", + "route": "auto", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 192, 64, 64], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [384, 192, 1, 1], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 384, 64, 64], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d pw_performance [1, 384, 192, 64, 64, False] float32 auto", + "operator": "aten.convolution.default", + "route": "auto", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 384, 64, 64], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [192, 384, 1, 1], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 192, 64, 64], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d pw_performance [1, 384, 192, 64, 64, False] float16 auto", + "operator": "aten.convolution.default", + "route": "auto", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 384, 64, 64], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [192, 384, 1, 1], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 192, 64, 64], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d pw_performance [1, 384, 768, 32, 32, False] float32 auto", + "operator": "aten.convolution.default", + "route": "auto", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 384, 32, 32], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [768, 384, 1, 1], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 768, 32, 32], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d pw_performance [1, 384, 768, 32, 32, False] float16 auto", + "operator": "aten.convolution.default", + "route": "auto", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 384, 32, 32], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [768, 384, 1, 1], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 768, 32, 32], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d pw_performance [1, 768, 384, 32, 32, False] float32 auto", + "operator": "aten.convolution.default", + "route": "auto", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 768, 32, 32], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [384, 768, 1, 1], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 384, 32, 32], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d pw_performance [1, 768, 384, 32, 32, False] float16 auto", + "operator": "aten.convolution.default", + "route": "auto", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 768, 32, 32], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [384, 768, 1, 1], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 384, 32, 32], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d pw_performance [1, 48, 256, 256, 256, False] float32 auto", + "operator": "aten.convolution.default", + "route": "auto", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 48, 256, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 48, 1, 1], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 256, 256, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d pw_performance [1, 48, 256, 256, 256, False] float16 auto", + "operator": "aten.convolution.default", + "route": "auto", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 48, 256, 256], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 48, 1, 1], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 256, 256, 256], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d pw_performance [1, 256, 32, 256, 256, False] float32 auto", + "operator": "aten.convolution.default", + "route": "auto", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 256, 256, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 256, 1, 1], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 256, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d pw_performance [1, 256, 32, 256, 256, False] float16 auto", + "operator": "aten.convolution.default", + "route": "auto", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 256, 256, 256], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 256, 1, 1], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 256, 256], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d pw_performance [1, 96, 256, 128, 128, False] float32 auto", + "operator": "aten.convolution.default", + "route": "auto", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 96, 128, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 96, 1, 1], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 256, 128, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d pw_performance [1, 96, 256, 128, 128, False] float16 auto", + "operator": "aten.convolution.default", + "route": "auto", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 96, 128, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 96, 1, 1], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 256, 128, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d pw_performance [1, 256, 64, 128, 128, False] float32 auto", + "operator": "aten.convolution.default", + "route": "auto", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 256, 128, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 256, 1, 1], + "dtype": "float32", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 128, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "conv2d pw_performance [1, 256, 64, 128, 128, False] float16 auto", + "operator": "aten.convolution.default", + "route": "auto", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 256, 128, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 256, 1, 1], + "dtype": "float16", + "storage": "constant", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "bool": false + }, + { + "int_list": [0, 0] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 128, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/Q8taConv2dReference.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/Q8taConv2dReference.cpp new file mode 100644 index 00000000000..dae94c71558 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/Q8taConv2dReference.cpp @@ -0,0 +1,146 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include +#include +#include + +#include +#include + +namespace vkcompute::shader_profiling::reference { +namespace { +bool fused_bias(const Json& spec) { + const Json context = spec.value("reference_context", Json::object()); + if (context.contains("shaders")) { + for (const Json& node : context.at("shaders")) { + if (node.get().rfind("q8ta_conv2d_pw", 0) == 0) + return true; + } + return false; + } + const Json route = spec.value("route", Json("auto")); + const std::string name = route.is_string() ? route.get() + : route.value("name", "auto"); + return spec.at("operator") == "et_vk.q8ta_conv2d_pw.default" || + name.rfind("im2col", 0) == 0; +} +} // namespace + +Tensor q8ta_conv2d_reference( + const Json& spec, + const TensorMap& tensors, + size_t threads) { + const Conv2dParameters p{ + conv2d_pair_arg(spec, 9), + conv2d_pair_arg(spec, 10), + conv2d_pair_arg(spec, 11), + conv2d_pair_arg(spec, 12), + q8ta::integer(spec, 13)}; + const float input_scale = q8ta::scale(spec, 1); + const float output_scale = q8ta::scale(spec, 6); + const int64_t input_zp = q8ta::integer(spec, 2); + const int64_t output_zp = q8ta::integer(spec, 7); + require( + std::isfinite(input_scale) && input_scale > 0 && + std::isfinite(output_scale) && output_scale > 0, + "Q8 convolution scales must be positive and finite"); + require( + input_zp >= -128 && input_zp <= 127 && output_zp >= -128 && + output_zp <= 127, + "Q8 convolution zero points must fit int8"); + const bool legacy = spec.at("args").size() == 15; + const std::string activation = + legacy ? "none" : spec.at("args").at(14).at("string").get(); + require( + activation == "none" || activation == "relu", + "Q8 convolution reference supports none/relu activation"); + const Tensor& input = tensor_arg(spec, 0, tensors); + const Tensor& weight = tensor_arg(spec, 3, tensors); + const Tensor& sums = tensor_arg(spec, 4, tensors); + const Tensor& scales = tensor_arg(spec, 5, tensors); + const Tensor* bias = + spec.at("args").at(8).is_null() ? nullptr : &tensor_arg(spec, 8, tensors); + const std::string output_name = spec.at("args").back().at("tensor"); + const auto output_sizes = spec.at("tensors") + .at(output_name) + .at("sizes") + .get>(); + require(output_sizes.size() == 4, "Q8 convolution output must be NCHW"); + const Tensor geometry_weight{{output_sizes[1]}, {}}; + const auto g = conv2d_geometry(input, geometry_weight, nullptr, p); + require( + output_sizes == + std::vector{ + g.batches, g.out_channels, g.out_height, g.out_width}, + "Q8 convolution output shape disagrees with parameters"); + const bool depthwise = weight.sizes.size() == 3; + require( + depthwise + ? (weight.sizes == + std::vector{ + p.kernel[0], p.kernel[1], g.out_channels} && + g.channels == g.out_channels && p.groups == g.channels) + : (weight.sizes.size() == 2 && weight.sizes[0] == g.out_channels && + weight.sizes[1] >= g.row_size), + "Q8 convolution weights require OHWI rows or depthwise HWC"); + require( + sums.values.size() >= size_t(g.out_channels) && + scales.values.size() >= size_t(g.out_channels) && + (!bias || bias->values.size() >= size_t(g.out_channels)), + "Q8 convolution requires per-output-channel sums/scales/bias"); + const auto weight_index = + [&](int64_t oc, int64_t ci, int64_t ky, int64_t kx) { + return depthwise ? (ky * p.kernel[1] + kx) * g.out_channels + oc + : oc * weight.sizes[1] + + (ky * p.kernel[1] + kx) * g.in_channels_per_group + ci; + }; + std::vector logical_sums(size_t(g.out_channels), 0); + for (int64_t oc = 0; oc < g.out_channels; ++oc) + for (int64_t ci = 0; ci < g.in_channels_per_group; ++ci) + for (int64_t ky = 0; ky < p.kernel[0]; ++ky) + for (int64_t kx = 0; kx < p.kernel[1]; ++kx) + logical_sums[size_t(oc)] += + int64_t(weight.values[size_t(weight_index(oc, ci, ky, kx))]); + const bool fused = !legacy && fused_bias(spec); + return convolve( + input, + weight, + p, + g, + threads, + weight_index, + [&](int64_t sum, double value, double weight_value) { + return sum + (int64_t(value) - input_zp) * int64_t(weight_value); + }, + [&](int64_t sum, int64_t oc) { + const int64_t raw = sum + input_zp * logical_sums[size_t(oc)]; + const float corrected = fused + ? float(raw - input_zp * int64_t(sums.values[size_t(oc)])) + : float(raw) - float(sums.values[size_t(oc)]) * float(input_zp); + const float multiplier = float(scales.values[size_t(oc)]) * input_scale; + const float channel_bias = bias ? float(bias->values[size_t(oc)]) : 0; + const float scaled = corrected * multiplier; + float value = fused ? std::fma(corrected, multiplier, channel_bias) + : scaled + channel_bias; + if (activation == "relu") + value = std::max(value, 0.0f); + return double(q8ta::requantize(value, 1.0f / output_scale, output_zp)); + }); +} + +REGISTER_REFERENCE_FUNCTIONS { + REGISTER_REFERENCE_FN(q8ta_conv2d, q8ta_conv2d_reference); + REGISTER_OPERATOR_REFERENCE(et_vk.q8ta_conv2d.default, q8ta_conv2d); + REGISTER_OPERATOR_REFERENCE(et_vk.q8ta_conv2d_general.default, q8ta_conv2d); + REGISTER_OPERATOR_REFERENCE(et_vk.q8ta_conv2d_im2col.default, q8ta_conv2d); + REGISTER_OPERATOR_REFERENCE(et_vk.q8ta_conv2d_dw.default, q8ta_conv2d); + REGISTER_OPERATOR_REFERENCE(et_vk.q8ta_conv2d_pw.default, q8ta_conv2d); + REGISTER_OPERATOR_REFERENCE( + et_vk.conv2d_q8ta_q8csw_q8to.default, q8ta_conv2d); +} +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/Q8taConv2dReference.h b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/Q8taConv2dReference.h new file mode 100644 index 00000000000..251f382d8d0 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/Q8taConv2dReference.h @@ -0,0 +1,21 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include + +namespace vkcompute::shader_profiling::reference { + +Tensor q8ta_conv2d_reference( + const Json& spec, + const TensorMap& tensors, + size_t threads); +Tensor q8ta_transposed_reference( + const Json& spec, + const TensorMap& tensors, + size_t threads); +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/Q8taConv2dRoutes.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/Q8taConv2dRoutes.cpp new file mode 100644 index 00000000000..8c017cf3279 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/Q8taConv2dRoutes.cpp @@ -0,0 +1,298 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace vkcompute::shader_profiling { +using namespace convolution; +namespace { + +ConvShape conv_shape(ComputeGraph& graph, const std::vector& args) { + if (args.size() != 16u) + throw std::invalid_argument( + "Convolution route received wrong argument count"); + const ValueRef input = args[0]; + const ValueRef output = args.back(); + const ValueRef weight = args[3]; + ConvShape shape = + conv_shape_base(graph, input, weight, output, graph.get_int(args[13]), 2); + + shape.kernel = spatial_parameter(graph, args[9], true); + const int64_t kernel_columns = product( + product(shape.input[1] / shape.groups, shape.kernel[0]), shape.kernel[1]); + supported( + shape.weight[1] == padded4(kernel_columns), + "Q8 convolution weights require aligned [Cout, Kh*Kw*Cin/groups] rows"); + check_conv_output(graph, shape, args[10], args[11], args[12]); + + supported( + graph.dtype_of(input) == vkapi::kInt8x4 && + graph.dtype_of(output) == vkapi::kInt8x4 && + graph.is_buffer_storage(input) && graph.is_buffer_storage(output), + "Q8 convolution requires Int8x4 buffer input/output"); + supported( + q8ta_conv2d_check_packed_dim_info(graph.packed_dim_info_of(input)) && + q8ta_conv2d_check_packed_dim_info(graph.packed_dim_info_of(output)), + "Q8 convolution requires channels-packed4 with outer width block 1 or 4"); + supported( + graph.dtype_of(weight) == vkapi::kChar, + "Q8 convolution weights require int8 dtype"); + for (size_t i : {4, 5, 8}) { + if (i == 8 && graph.val_is_none(args[i])) { + continue; + } + constant_tensor(graph, args[i], "Q8 convolution channel parameters"); + supported( + graph.dtype_of(args[i]) == (i == 4 ? vkapi::kInt : vkapi::kFloat) && + graph.dim_of(args[i]) == 1 && + graph.size_at(0, args[i]) >= shape.output[1], + "Q8 convolution sums/scales/bias require canonical int32/float32 channel vectors"); + } + for (size_t i : {1, 6}) { + const double scale = graph.extract_scalar(args[i]); + supported( + std::isfinite(scale) && scale > 0, + "Q8 convolution scales must be positive and finite"); + } + for (size_t i : {2, 7}) { + const int64_t zero_point = graph.extract_scalar(args[i]); + supported( + zero_point >= -128 && zero_point <= 127, + "Q8 convolution zero points must fit int8"); + } + const std::string activation = graph.extract_string(args[14]); + supported( + activation == "none" || activation == "relu", + "Q8 convolution supports none/relu activation"); + return shape; +} + +} // namespace + +Json build_q8ta_conv2d_direct( + ComputeGraph& graph, + const std::vector& args, + const Json& options) { + check_options(options); + const ConvShape shape = conv_shape(graph, args); + grouped_packing(shape); + q8ta_conv2d_general_impl(graph, args); + return selected("direct"); +} + +namespace { +enum class DotRoute { Auto, Signed, Unsigned, Fallback }; + +Json build_im2col( + ComputeGraph& graph, + const std::vector& args, + const Json& options, + DotRoute dot, + const char* route_name) { + check_options(options); + const ConvShape shape = conv_shape(graph, args); + grouped_packing(shape); + supported( + shape.input[1] / shape.groups % 4 == 0, + "Q8 im2col convolution requires input channels per group divisible by4"); + const auto sizes = + calculate_q8ta_im2col_sizes(&graph, args[0], args[15], args[9], args[13]); + const auto plan = make_q8ta_conv2d_stream_plan_for_device( + sizes[0], sizes[1], sizes[2], sizes[3], graph.max_buffer_numel()); + supported(plan.feasible, "Q8 im2col has no feasible device scratch plan"); + if (dot == DotRoute::Unsigned) { + supported( + graph.can_use_int8_dot_product() && + shape.weight[1] <= kMaxUnsignedDotAccumulatorBytes, + "Unsigned Q8 im2col requires packed integer dot and bounded K"); + q8ta_conv2d_im2col_impl(graph, true, args); + } else if (dot == DotRoute::Signed) { + q8ta_conv2d_im2col_impl(graph, false, args); + } else if (dot == DotRoute::Fallback) { + vkapi::ScopedAdapterCapabilityOverride no_dot( + graph.context()->adapter_ptr(), + vkapi::AdapterCapabilityOverrides::without_dot_product_support()); + q8ta_conv2d_im2col_impl(graph, false, args); + } else { + q8ta_conv2d_im2col(graph, args); + } + return selected(route_name); +} + +Json build_im2col_signed( + ComputeGraph& graph, + const std::vector& args, + const Json& options) { + return build_im2col(graph, args, options, DotRoute::Signed, "im2col"); +} +Json build_im2col_unsigned( + ComputeGraph& graph, + const std::vector& args, + const Json& options) { + return build_im2col( + graph, args, options, DotRoute::Unsigned, "im2col_unsigned"); +} +Json build_im2col_fallback( + ComputeGraph& graph, + const std::vector& args, + const Json& options) { + return build_im2col( + graph, args, options, DotRoute::Fallback, "im2col_fallback"); +} + +Json build_pointwise( + ComputeGraph& graph, + const std::vector& args, + const Json& options, + DotRoute dot, + const char* route_name) { + check_options(options); + const ConvShape shape = conv_shape(graph, args); + supported( + shape.kernel == std::array{1, 1} && shape.groups == 1 && + spatial_parameter(graph, args[10], true) == + std::array{1, 1} && + spatial_parameter(graph, args[11], false) == + std::array{0, 0}, + "Q8 pointwise route requires ungrouped1x1, unit stride, zero padding"); + if (dot == DotRoute::Unsigned) { + supported( + graph.can_use_int8_dot_product() && + shape.weight[1] <= kMaxUnsignedDotAccumulatorBytes, + "Unsigned Q8 pointwise requires packed integer dot and bounded K"); + q8ta_conv2d_pw_impl(graph, true, args); + } else if (dot == DotRoute::Signed) { + q8ta_conv2d_pw_impl(graph, false, args); + } else { + q8ta_conv2d_pw(graph, args); + } + return selected(route_name); +} +Json build_pw_signed( + ComputeGraph& graph, + const std::vector& args, + const Json& options) { + return build_pointwise(graph, args, options, DotRoute::Signed, "pw_signed"); +} +Json build_pw_unsigned( + ComputeGraph& graph, + const std::vector& args, + const Json& options) { + return build_pointwise( + graph, args, options, DotRoute::Unsigned, "pw_unsigned"); +} +Json build_pw_auto( + ComputeGraph& graph, + const std::vector& args, + const Json& options) { + return build_pointwise(graph, args, options, DotRoute::Auto, "pw_auto"); +} + +Json build_legacy_4w4c( + ComputeGraph& graph, + const std::vector& args, + const Json& options) { + check_options(options); + supported(args.size() == 16, "Q8 legacy route requires16 native arguments"); + for (ValueRef tensor : {args[0], args[15]}) { + const auto& packing = graph.packed_dim_info_of(tensor); + supported( + graph.dtype_of(tensor) == vkapi::kInt8x4 && + graph.is_buffer_storage(tensor) && + q8ta_conv2d_check_packed_dim_info(packing) && + packing.outer_packed_dim_block_size == 4, + "Q8 legacy route requires4w4c input/output buffers"); + } + supported( + graph.extract_string(args[14]) == "none", + "Q8 legacy route has no fused activation argument"); + std::vector native(args.begin(), args.begin() + 14); + native.push_back(args[15]); + VK_GET_OP_FN("et_vk.conv2d_q8ta_q8csw_q8to.default")(graph, native); + return selected("legacy_4w4c"); +} +} // namespace + +Json build_q8ta_conv2d_im2col( + ComputeGraph& graph, + const std::vector& args, + const Json& options) { + return build_im2col(graph, args, options, DotRoute::Auto, "im2col"); +} + +Json inspect_q8ta_conv2d_route( + ComputeGraph& graph, + const std::vector&, + size_t first, + size_t end) { + for (size_t i = first; i < end; ++i) { + const std::string name = graph.execute_nodes().at(i)->name(); + if (name == "q8ta_im2col") { + return selected("im2col"); + } + if (name.rfind("q8ta_conv2d_pw", 0) == 0) { + return selected("pointwise"); + } + if (name.rfind("q8ta_conv2d_dw", 0) == 0) { + return selected("depthwise"); + } + if (name.rfind("q8ta_conv2d_transposed", 0) == 0) { + return selected("transposed"); + } + if (name == "q8ta_conv2d_float" || name == "q8ta_conv2d_fallback_float") { + return selected("direct"); + } + } + return Json(); +} + +REGISTER_OPERATOR_ROUTES { + REGISTER_OPERATOR_ROUTE( + et_vk.q8ta_conv2d.default, direct, build_q8ta_conv2d_direct); + REGISTER_OPERATOR_ROUTE( + et_vk.q8ta_conv2d.default, general, build_q8ta_conv2d_direct); + REGISTER_OPERATOR_ROUTE( + et_vk.q8ta_conv2d.default, im2col, build_im2col_signed); + REGISTER_OPERATOR_ROUTE( + et_vk.q8ta_conv2d.default, im2col_auto, build_q8ta_conv2d_im2col); + REGISTER_OPERATOR_ROUTE( + et_vk.q8ta_conv2d.default, im2col_unsigned, build_im2col_unsigned); + REGISTER_OPERATOR_ROUTE( + et_vk.q8ta_conv2d.default, im2col_fallback, build_im2col_fallback); + REGISTER_OPERATOR_ROUTE( + et_vk.q8ta_conv2d.default, legacy_4w4c, build_legacy_4w4c); + REGISTER_OPERATOR_ROUTE( + et_vk.q8ta_conv2d_dw.default, legacy_4w4c, build_legacy_4w4c); + REGISTER_OPERATOR_ROUTE( + et_vk.q8ta_conv2d_pw.default, legacy_4w4c, build_legacy_4w4c); + REGISTER_OPERATOR_ROUTE( + et_vk.q8ta_conv2d_pw.default, pw_signed, build_pw_signed); + REGISTER_OPERATOR_ROUTE( + et_vk.q8ta_conv2d_pw.default, pw_unsigned, build_pw_unsigned); + REGISTER_OPERATOR_ROUTE(et_vk.q8ta_conv2d_pw.default, pw_auto, build_pw_auto); + REGISTER_ROUTE_INSPECTOR( + et_vk.q8ta_conv2d.default, inspect_q8ta_conv2d_route); + REGISTER_ROUTE_INSPECTOR( + et_vk.q8ta_conv2d_general.default, inspect_q8ta_conv2d_route); + REGISTER_ROUTE_INSPECTOR( + et_vk.q8ta_conv2d_im2col.default, inspect_q8ta_conv2d_route); + REGISTER_ROUTE_INSPECTOR( + et_vk.q8ta_conv2d_dw.default, inspect_q8ta_conv2d_route); + REGISTER_ROUTE_INSPECTOR( + et_vk.q8ta_conv2d_pw.default, inspect_q8ta_conv2d_route); + REGISTER_ROUTE_INSPECTOR( + et_vk.q8ta_conv2d_transposed.default, inspect_q8ta_conv2d_route); +} +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/Q8taConv2dRoutes.h b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/Q8taConv2dRoutes.h new file mode 100644 index 00000000000..ca9f8163d5a --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/Q8taConv2dRoutes.h @@ -0,0 +1,26 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include + +namespace vkcompute::shader_profiling { +Json build_q8ta_conv2d_direct( + ComputeGraph& graph, + const std::vector& args, + const Json& options); +Json build_q8ta_conv2d_im2col( + ComputeGraph& graph, + const std::vector& args, + const Json& options); +Json inspect_q8ta_conv2d_route( + ComputeGraph& graph, + const std::vector& args, + size_t first, + size_t end); +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/Q8taConv2dThroughput.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/Q8taConv2dThroughput.cpp new file mode 100644 index 00000000000..a18f440541f --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/Q8taConv2dThroughput.cpp @@ -0,0 +1,122 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include + +namespace vkcompute::shader_profiling { +namespace { +double q8ta_conv2d_flops(const Json& spec) { + const auto input = throughput::tensor_argument(spec, 0) + .at("sizes") + .get>(); + const auto weight = throughput::tensor_argument(spec, 3) + .at("sizes") + .get>(); + const Json& output_tensor = throughput::output_tensor(spec); + const auto output = output_tensor.at("sizes").get>(); + const auto kernel = + spec.at("args").at(9).at("int_list").get>(); + const bool transposed = + spec.value("operator_name", spec.value("operator", "")) == + "et_vk.q8ta_conv2d_transposed.default"; + const int64_t groups = spec.at("args").at(transposed ? 14 : 13).at("int"); + if (input.size() != 4 || output.size() != 4 || kernel.size() != 2 || + groups <= 0 || input[1] % groups || output[1] % groups || + input[0] != output[0] || kernel[0] <= 0 || kernel[1] <= 0) + throw std::invalid_argument( + "Q8 convolution work count dimensions disagree"); + const int64_t logical_columns = input[1] / groups * kernel[0] * kernel[1]; + const int64_t padded_columns = (logical_columns + 3) / 4 * 4; + if (weight.size() == 3) { + if (weight != std::vector{kernel[0], kernel[1], output[1]} || + groups != input[1] || input[1] != output[1]) + throw std::invalid_argument( + "Q8 depthwise work count requires HWC weights"); + } else if ( + weight.size() != 2 || weight[0] != output[1] || + weight[1] != padded_columns) { + throw std::invalid_argument("Q8 work count requires aligned OHWI rows"); + } + const auto stride = + spec.at("args").at(10).at("int_list").get>(); + const auto pad = + spec.at("args").at(11).at("int_list").get>(); + const auto dilation = spec.at("args") + .at(transposed ? 13 : 12) + .at("int_list") + .get>(); + const auto output_pad = transposed + ? spec.at("args").at(12).at("int_list").get>() + : std::vector{0, 0}; + if (stride.size() != 2 || pad.size() != 2 || dilation.size() != 2 || + output_pad.size() != 2) + throw std::invalid_argument("Q8 work count requires spatial pairs"); + for (size_t d = 0; d < 2; ++d) { + if (stride[d] <= 0 || pad[d] < 0 || dilation[d] <= 0) + throw std::invalid_argument( + "Q8 work count spatial parameters are invalid"); + const int64_t expected = transposed + ? (input[d + 2] - 1) * stride[d] - 2 * pad[d] + + dilation[d] * (kernel[d] - 1) + output_pad[d] + 1 + : (input[d + 2] + 2 * pad[d] - dilation[d] * (kernel[d] - 1) - 1) / + stride[d] + + 1; + if (expected != output[d + 2]) + throw std::invalid_argument("Q8 work count output shape disagrees"); + } + const double elements = throughput::tensor_elements(output_tensor); + throughput::tensor_elements(throughput::tensor_argument(spec, 0)); + throughput::tensor_elements(throughput::tensor_argument(spec, 3)); + const bool bias = !spec.at("args").at(8).is_null(); + if (bias && + throughput::tensor_elements(throughput::tensor_argument(spec, 8)) < + double(output[1])) + throw std::invalid_argument( + "Q8 work count bias must cover output channels"); + const double multiply_adds = transposed + ? 2.0 * double(input[0]) * double(input[1]) * double(input[2]) * + double(input[3]) * double(output[1] / groups) * double(kernel[0]) * + double(kernel[1]) + : elements * 2.0 * double(logical_columns); + return multiply_adds + (bias ? elements : 0); +} +} // namespace + +REGISTER_THROUGHPUT_FUNCTIONS { + REGISTER_OPERATOR_MEMORY_TRAFFIC( + et_vk.q8ta_conv2d.default, throughput::tensor_traffic); + REGISTER_OPERATOR_MEMORY_TRAFFIC( + et_vk.q8ta_conv2d_general.default, throughput::tensor_traffic); + REGISTER_OPERATOR_MEMORY_TRAFFIC( + et_vk.q8ta_conv2d_im2col.default, throughput::tensor_traffic); + REGISTER_OPERATOR_MEMORY_TRAFFIC( + et_vk.q8ta_conv2d_dw.default, throughput::tensor_traffic); + REGISTER_OPERATOR_MEMORY_TRAFFIC( + et_vk.q8ta_conv2d_pw.default, throughput::tensor_traffic); + REGISTER_OPERATOR_MEMORY_TRAFFIC( + et_vk.q8ta_conv2d_transposed.default, throughput::tensor_traffic); + REGISTER_OPERATOR_MEMORY_TRAFFIC( + et_vk.conv2d_q8ta_q8csw_q8to.default, throughput::tensor_traffic); + REGISTER_OPERATOR_COMPUTE_FLOPS(et_vk.q8ta_conv2d.default, q8ta_conv2d_flops); + REGISTER_OPERATOR_COMPUTE_FLOPS( + et_vk.q8ta_conv2d_general.default, q8ta_conv2d_flops); + REGISTER_OPERATOR_COMPUTE_FLOPS( + et_vk.q8ta_conv2d_im2col.default, q8ta_conv2d_flops); + REGISTER_OPERATOR_COMPUTE_FLOPS( + et_vk.q8ta_conv2d_dw.default, q8ta_conv2d_flops); + REGISTER_OPERATOR_COMPUTE_FLOPS( + et_vk.q8ta_conv2d_pw.default, q8ta_conv2d_flops); + REGISTER_OPERATOR_COMPUTE_FLOPS( + et_vk.q8ta_conv2d_transposed.default, q8ta_conv2d_flops); + REGISTER_OPERATOR_COMPUTE_FLOPS( + et_vk.conv2d_q8ta_q8csw_q8to.default, q8ta_conv2d_flops); + REGISTER_SHADER_MEMORY_TRAFFIC(q8ta_conv2d, throughput::tensor_traffic); + REGISTER_SHADER_MEMORY_TRAFFIC(q8ta_im2col, throughput::tensor_traffic); + REGISTER_SHADER_MEMORY_TRAFFIC(conv2d_dw_q8ta, throughput::tensor_traffic); +} +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/Q8taValidationChecks.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/Q8taValidationChecks.cpp new file mode 100644 index 00000000000..d7f667a9fe6 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/Q8taValidationChecks.cpp @@ -0,0 +1,158 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include +#include +#include +#include +#include +#include +#include +namespace vkcompute::shader_profiling { +Json check_q8ta_unsigned_dot_selection(const Json& spec, ComputeGraph& graph) { + const auto& parameters = spec.at("parameters"); + const int64_t expected_limit = parameters.at("accumulator_limit_bytes"); + reference::require( + expected_limit == kMaxUnsignedDotAccumulatorBytes, + "Unsigned dot accumulator boundary changed"); + const auto& adapter = *graph.context()->adapter_ptr(); + const bool prefers_unsigned = adapter.accelerates_unsigned_packed4x8_dot() && + !adapter.accelerates_signed_packed4x8_dot(); + Json decisions = Json::array(); + for (const Json& size : parameters.at("test_kernel_sizes")) { + const int64_t k = size.get(); + const bool selected = can_use_unsigned_pw_dot(adapter, k); + reference::require( + selected == (prefers_unsigned && k <= expected_limit), + "Unsigned dot selection violated capability or accumulator limit"); + decisions.push_back({{"kernel_bytes", k}, {"unsigned_selected", selected}}); + } + return { + {"status", "PASS"}, + {"prefers_unsigned", prefers_unsigned}, + {"decisions", decisions}}; +} +Json check_q8ta_streaming_dynamic_shrink( + const Json& spec, + ComputeGraph& graph) { + const Json& parameters = spec.at("parameters"); + const Json& plan_args = parameters.at("stream_plan"); + const auto plan = make_q8ta_conv2d_stream_plan( + plan_args.at("batch").get(), + plan_args.at("flattened_kernel_size").get(), + plan_args.at("out_height").get(), + plan_args.at("out_width").get(), + plan_args.at("scratch_budget_bytes").get()); + const Json& expected = parameters.at("expected_plan"); + reference::require( + plan.feasible && + plan.num_tiles == expected.at("num_tiles").get() && + plan.rows_per_tile >= expected.at("min_rows_per_tile").get(), + "Dynamic shrink case did not create dead stream tiles"); + const int64_t resized_bytes = + plan_args.at("flattened_kernel_size").get() * + plan.rows_per_tile * + utils::align_up_4(parameters.at("resized_width").get()); + reference::require( + resized_bytes <= plan_args.at("scratch_budget_bytes").get(), + "Resized im2col scratch exceeds stream budget"); + Json operator_spec = parse_configs(parameters.at("operator_case")).front(); + const std::string output_name = parameters.at("output_name"); + Json input_spec = operator_spec; + input_spec["tensors"].erase(output_name); + Json output_spec = operator_spec; + output_spec["tensors"] = { + {output_name, operator_spec.at("tensors").at(output_name)}}; + auto host_data = prepare_tensor_data(input_spec); + const auto input_tensors = create_tensors(input_spec, graph, host_data, true); + auto output_data = prepare_tensor_data(output_spec); + const auto output_tensors = + create_tensors(output_spec, graph, output_data, false); + host_data.merge(output_data); + auto tensors = input_tensors; + tensors.insert(output_tensors.begin(), output_tensors.end()); + for (const auto& input : graph.inputs()) + reference::require( + input.value != tensors.at(output_name), + "Dynamic validation output must not be staged as an input"); + reference::require( + graph.dtype_of(tensors.at( + parameters.at("input_name").get())) == vkapi::kInt8x4 && + graph.dtype_of(tensors.at(output_name)) == vkapi::kInt8x4, + "Dynamic Q8 convolution requires native int8x4 input/output"); + const auto args = build_operator_args(operator_spec, graph, tensors); + build_operator_route(operator_spec, graph, args); + Json accesses = Json::object(); + for (const auto& tensor : tensors) + accesses[tensor.first] = tensor.first == output_name ? "write" : "read"; + const auto staging = + add_validation_outputs(operator_spec, graph, tensors, accesses); + graph.prepare(); + graph.prepack(); + Json validations = Json::array(); + for (const Json& step : spec.at("validation_steps")) { + Json resized = operator_spec; + auto data = host_data; + const Json& sizes = step.at("tensor_sizes"); + reference::require( + !sizes.contains(output_name), + "Dynamic validation steps must not resize the output manually"); + for (const auto& item : sizes.items()) { + const auto shape = item.value().get>(); + const ValueRef ref = input_tensors.at(item.key()); + bool found = false; + for (size_t i = 0; i < graph.inputs().size(); ++i) + if (graph.inputs().at(i).value == ref) { + graph.resize_input(i, shape); + found = true; + break; + } + reference::require( + found, "Dynamic validation size target is not a staged input"); + resized["tensors"][item.key()]["sizes"] = shape; + const size_t bytes = tensor_numel(item.value()); + reference::require( + bytes <= data.at(item.key()).size(), + "Dynamic validation shape exceeds upper input allocation"); + data.at(item.key()).resize(bytes); + } + resized["tensors"][output_name]["sizes"] = step.at("expected_output_sizes"); + graph.propagate_resize(); + reference::require( + graph.sizes_of(tensors.at(output_name)) == + step.at("expected_output_sizes").get>(), + "Streaming im2col output did not resize"); + const size_t output_bytes = tensor_numel(step.at("expected_output_sizes")); + reference::require( + output_bytes <= data.at(output_name).size(), + "Dynamic validation output exceeds upper allocation"); + data.at(output_name).resize(output_bytes); + copy_validation_inputs(resized, graph, input_tensors, data); + graph.execute(); + if (step.value("validate_output", true)) { + const Json validation = + validate_outputs(resized, graph, tensors, staging, data); + reference::require( + validation.at("status") == "PASS", + "Streaming im2col resize validation failed: " + validation.dump()); + validations.push_back(validation); + } + } + return { + {"status", "PASS"}, + {"validation_steps", spec.at("validation_steps").size()}, + {"validated_outputs", validations.size()}, + {"stream_tiles", plan.num_tiles}, + {"resized_scratch_bytes", resized_bytes}}; +} +REGISTER_CASE_CHECKS { + REGISTER_CASE_CHECK( + q8ta_unsigned_dot_selection, check_q8ta_unsigned_dot_selection); + REGISTER_CASE_CHECK( + q8ta_streaming_dynamic_shrink, check_q8ta_streaming_dynamic_shrink); +} +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/Q8taValidationChecks.h b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/Q8taValidationChecks.h new file mode 100644 index 00000000000..845ef6fb368 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/Q8taValidationChecks.h @@ -0,0 +1,13 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once +#include +#include +namespace vkcompute::shader_profiling { +Json check_q8ta_unsigned_dot_selection(const Json& spec, ComputeGraph& graph); +Json check_q8ta_streaming_dynamic_shrink(const Json& spec, ComputeGraph& graph); +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/correctness.json b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/correctness.json new file mode 100644 index 00000000000..19f3961636d --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/correctness.json @@ -0,0 +1,3474 @@ +[ + { + "name": "q8ta/conv2d/correctness int8x4 buffer general", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "general", + "sweep_variables": { + "v_LAYOUT": ["4c1w", "4w4c", "4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 3, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 28], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 27 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/correctness int8x4 buffer auto", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "sweep_variables": { + "v_LAYOUT": ["4c1w", "4w4c", "4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 3, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 28], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 27 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/correctness [3]", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "legacy_4w4c", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 3, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 28], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 27 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/correctness int8x4 buffer 4c1w", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/correctness int8x4 buffer 4w4c", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "legacy_4w4c", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/correctness int8x4 buffer 4c", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/correctness int8x4 buffer 4c1w (6)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/correctness int8x4 buffer 4w4c (7)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "legacy_4w4c", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/correctness int8x4 buffer 4c (8)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/correctness int8x4 buffer 4c1w (9)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 77, 77], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [16, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [16], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 77, 77], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/correctness int8x4 buffer 4w4c (10)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "legacy_4w4c", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 77, 77], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [16, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [16], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 77, 77], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/correctness int8x4 buffer 4c (11)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 77, 77], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [16, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [16], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 77, 77], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/correctness int8x4 buffer", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "im2col"], + "v_LAYOUT": ["4c1w", "4w4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [2, 32, 7, 7], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [16, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [16], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [2, 16, 7, 7], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/correctness int8x4 buffer 4c1w (13)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 144], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 2 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/correctness int8x4 buffer 4w4c (14)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "legacy_4w4c", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 144], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 2 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/correctness int8x4 buffer 4c (15)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 144], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 2 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/correctness int8x4 buffer 4c1w (16)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 3, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 28], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 27 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 32, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/correctness int8x4 buffer 4c1w (17)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 32, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/correctness int8x4 buffer 4w4c (18)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "legacy_4w4c", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 3, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 28], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 27 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 32, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/correctness int8x4 buffer 4w4c (19)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "legacy_4w4c", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 32, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/correctness int8x4 buffer 4c (20)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 3, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 28], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 27 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 32, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/correctness int8x4 buffer 4c (21)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 32, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/correctness int8x4 buffer 4c1w (22)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 96, 81, 81], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [96, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [96], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [96], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [96], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 3 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 96, 41, 41], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/correctness int8x4 buffer 4w4c (23)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "legacy_4w4c", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 96, 81, 81], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [96, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [96], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [96], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [96], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 3 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 96, 41, 41], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/correctness int8x4 buffer 4c (24)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 96, 81, 81], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [96, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [96], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [96], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [96], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 3 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 96, 41, 41], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/correctness int8x4 buffer 4c1w (25)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 96, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [96, 600], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [96], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [96], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [96], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [5, 5] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int": 4 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 96, 32, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/correctness int8x4 buffer 4w4c (26)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "legacy_4w4c", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 96, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [96, 600], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [96], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [96], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [96], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [5, 5] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int": 4 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 96, 32, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/correctness int8x4 buffer 4c (27)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 96, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [96, 600], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [96], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [96], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [96], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [5, 5] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int": 4 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 96, 32, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/debug.json b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/debug.json new file mode 100644 index 00000000000..dbf8316980a --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/debug.json @@ -0,0 +1,368 @@ +[ + { + "name": "q8ta/conv2d/debug int8x4 buffer auto", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "sweep_variables": { + "v_LAYOUT": ["4c1w", "4w4c", "4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 8, 5, 5], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [16, 72], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [16], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 5, 5], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/debug int8x4 buffer im2col", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "im2col", + "sweep_variables": { + "v_LAYOUT": ["4c1w", "4w4c", "4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 8, 5, 5], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [16, 72], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [16], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 5, 5], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/debug [4]", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "legacy_4w4c", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 8, 5, 5], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [16, 72], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [16], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 5, 5], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/dw_correctness.json b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/dw_correctness.json new file mode 100644 index 00000000000..65e007ae390 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/dw_correctness.json @@ -0,0 +1,1826 @@ +[ + { + "name": "q8ta/conv2d/dw_correctness", + "operator": "et_vk.q8ta_conv2d_dw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "sweep_variables": { + "v_LAYOUT": ["4c1w", "4w4c", "4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [3, 3, 32], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "column_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 32 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/dw_correctness (2)", + "operator": "et_vk.q8ta_conv2d_dw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "sweep_variables": { + "v_LAYOUT": ["4c1w", "4w4c", "4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 80, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [3, 3, 80], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [80], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "column_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [80], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [80], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 80 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 80, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/dw_correctness (3)", + "operator": "et_vk.q8ta_conv2d_dw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "sweep_variables": { + "v_LAYOUT": ["4c1w", "4w4c", "4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 57, 33], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [3, 3, 16], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [16], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "column_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 16 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 57, 33], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/dw_correctness (4)", + "operator": "et_vk.q8ta_conv2d_dw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "sweep_variables": { + "v_LAYOUT": ["4c1w", "4w4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [2, 8, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [3, 3, 8], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [8], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "column_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [8], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [8], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 8 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [2, 8, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/dw_correctness (5)", + "operator": "et_vk.q8ta_conv2d_dw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "sweep_variables": { + "v_LAYOUT": ["4c1w", "4w4c", "4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [5, 5, 32], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "column_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [5, 5] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int": 32 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/dw_correctness (6)", + "operator": "et_vk.q8ta_conv2d_dw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "sweep_variables": { + "v_LAYOUT": ["4c1w", "4w4c", "4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 32, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [3, 3, 64], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "column_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int": 64 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 17, 17], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/dw_correctness (7)", + "operator": "et_vk.q8ta_conv2d_dw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "sweep_variables": { + "v_LAYOUT": ["4c1w", "4w4c", "4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 96, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [5, 5, 96], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [96], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "column_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [96], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [96], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [5, 5] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int": 96 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 96, 32, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/dw_correctness (8)", + "operator": "et_vk.q8ta_conv2d_dw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "sweep_variables": { + "v_LAYOUT": ["4c1w", "4w4c", "4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 32, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [3, 3, 64], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "column_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 64 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/dw_correctness [2]", + "operator": "et_vk.q8ta_conv2d_dw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "legacy_4w4c", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [3, 3, 32], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "column_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 32 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/dw_correctness [14]", + "operator": "et_vk.q8ta_conv2d_dw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "legacy_4w4c", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 80, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [3, 3, 80], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [80], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "column_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [80], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [80], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 80 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 80, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/dw_correctness [18]", + "operator": "et_vk.q8ta_conv2d_dw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "legacy_4w4c", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 57, 33], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [3, 3, 16], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [16], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "column_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 16 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 57, 33], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/dw_correctness [22]", + "operator": "et_vk.q8ta_conv2d_dw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "legacy_4w4c", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [5, 5, 32], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "column_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [5, 5] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int": 32 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/dw_correctness [6]", + "operator": "et_vk.q8ta_conv2d_dw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "legacy_4w4c", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 32, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [3, 3, 64], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "column_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int": 64 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 17, 17], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/dw_correctness [26]", + "operator": "et_vk.q8ta_conv2d_dw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "legacy_4w4c", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 96, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [5, 5, 96], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [96], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "column_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [96], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [96], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [5, 5] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int": 96 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 96, 32, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/dw_correctness [10]", + "operator": "et_vk.q8ta_conv2d_dw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "legacy_4w4c", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 32, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [3, 3, 64], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "column_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 64 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/dw_debug.json b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/dw_debug.json new file mode 100644 index 00000000000..0dad77a1b68 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/dw_debug.json @@ -0,0 +1,245 @@ +[ + { + "name": "q8ta/conv2d/dw_debug", + "operator": "et_vk.q8ta_conv2d_dw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "sweep_variables": { + "v_LAYOUT": ["4c1w", "4w4c", "4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 8, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [3, 3, 8], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [8], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "column_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [8], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [8], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 8 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 8, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/dw_debug [2]", + "operator": "et_vk.q8ta_conv2d_dw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "legacy_4w4c", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 8, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [3, 3, 8], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [8], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "column_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [8], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [8], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 8 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 8, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/dw_narrow_workgroup.json b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/dw_narrow_workgroup.json new file mode 100644 index 00000000000..d2d994c03da --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/dw_narrow_workgroup.json @@ -0,0 +1,248 @@ +[ + { + "name": "q8ta/conv2d/dw_narrow_workgroup [0]", + "operator": "et_vk.q8ta_conv2d_dw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128, 7, 7], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [3, 3, 128], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "column_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 128 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 128, 7, 7], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/dw_narrow_workgroup", + "operator": "et_vk.q8ta_conv2d_dw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "sweep_variables": { + "v_SIZES": [ + [1, 64, 9, 9], + [1, 64, 13, 13] + ] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": "v_SIZES", + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [3, 3, 64], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "column_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 64 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": "v_SIZES", + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/dw_performance.json b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/dw_performance.json new file mode 100644 index 00000000000..b0c7e7917f3 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/dw_performance.json @@ -0,0 +1,1214 @@ +[ + { + "name": "q8ta/conv2d/dw_performance", + "operator": "et_vk.q8ta_conv2d_dw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "sweep_variables": { + "v_LAYOUT": ["4c1w", "4w4c", "4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [3, 3, 128], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "column_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 128 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 128, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/dw_performance (2)", + "operator": "et_vk.q8ta_conv2d_dw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "sweep_variables": { + "v_LAYOUT": ["4c1w", "4w4c", "4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 256, 256], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [3, 3, 64], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "column_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 64 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 256, 256], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/dw_performance (3)", + "operator": "et_vk.q8ta_conv2d_dw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "sweep_variables": { + "v_LAYOUT": ["4c1w", "4w4c", "4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 288, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [3, 3, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [288], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "column_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [288], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [288], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 288 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 288, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/dw_performance (4)", + "operator": "et_vk.q8ta_conv2d_dw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "sweep_variables": { + "v_LAYOUT": ["4c1w", "4w4c", "4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [3, 3, 32], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "column_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int": 32 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 130, 130], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/dw_performance [16]", + "operator": "et_vk.q8ta_conv2d_dw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [5, 5, 128], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "column_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [5, 5] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int": 128 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 128, 32, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/dw_performance [17]", + "operator": "et_vk.q8ta_conv2d_dw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [60, 128, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [5, 5, 128], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "column_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [5, 5] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int": 128 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [60, 128, 32, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/dw_performance [2]", + "operator": "et_vk.q8ta_conv2d_dw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "legacy_4w4c", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [3, 3, 128], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "column_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 128 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 128, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/dw_performance [6]", + "operator": "et_vk.q8ta_conv2d_dw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "legacy_4w4c", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 256, 256], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [3, 3, 64], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "column_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 64 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 256, 256], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/dw_performance [10]", + "operator": "et_vk.q8ta_conv2d_dw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "legacy_4w4c", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 288, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [3, 3, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [288], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "column_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [288], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [288], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 288 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 288, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/dw_performance [14]", + "operator": "et_vk.q8ta_conv2d_dw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "legacy_4w4c", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [3, 3, 32], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "column_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int": 32 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 130, 130], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/im2col.json b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/im2col.json new file mode 100644 index 00000000000..888dbfd8844 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/im2col.json @@ -0,0 +1,961 @@ +[ + { + "name": "q8ta/conv2d/im2col", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.25, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["im2col", "im2col_unsigned", "im2col_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 4, 5, 5], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "repeat": [-128, -1, 0, 1, 127] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [5, 36], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [-1, 127, 0, -128, 1] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [8], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [8], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0.00390625] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [8], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 5, 3, 3], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/im2col (2)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.25, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["im2col_unsigned", "im2col_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 4, 1, 1], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [262145, 4], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [262148], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [262148], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0.00390625] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [262148], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 262145, 1, 1], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/im2col [19]", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.25, + "rtol": 0 + }, + "route": "im2col_auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 4, 91, 91], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [1, 33124], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0.00390625] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [91, 91] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 1, 1, 1], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/im2col (4)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.25, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["im2col", "im2col_unsigned", "im2col_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 4, 90, 91], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "repeat": [127] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 0 + }, + { + "tensor": { + "name": "weight", + "sizes": [1, 32760], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [9.999999974752427e-7] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [90, 91] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 1, 1, 1], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/im2col (5)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.25, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["im2col", "im2col_unsigned", "im2col_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 8, 6, 7], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "repeat": [-128, -1, 0, 1, 127] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 11 + }, + { + "tensor": { + "name": "weight", + "sizes": [8, 36], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [-1, 127, 0, -128, 1] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [8], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [8], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0.00390625] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -3 + }, + { + "tensor": { + "name": "bias", + "sizes": [8], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 2 + }, + { + "string": "relu" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 8, 6, 7], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/im2col (6)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.25, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["im2col", "im2col_unsigned", "im2col_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 8, 7, 7], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "repeat": [-128, -1, 0, 1, 127] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 127 + }, + { + "tensor": { + "name": "weight", + "sizes": [12, 72], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [-1, 127, 0, -128, 1] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [12], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [12], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0.00390625] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -5 + }, + { + "tensor": { + "name": "bias", + "sizes": [12], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "relu" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 12, 4, 4], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/im2col (7)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.25, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["im2col", "im2col_unsigned", "im2col_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 8, 5, 5], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "repeat": [-128, -1, 0, 1, 127] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": -7 + }, + { + "tensor": { + "name": "weight", + "sizes": [8, 72], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [-1, 127, 0, -128, 1] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [8], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [8], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0.00390625] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": 3 + }, + null, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 8, 5, 5], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/im2col (8)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.25, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["im2col", "im2col_unsigned", "im2col_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 8, 9, 9], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "repeat": [-128, -1, 0, 1, 127] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": -128 + }, + { + "tensor": { + "name": "weight", + "sizes": [12, 72], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [-1, 127, 0, -128, 1] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [12], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [12], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0.00390625] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": 5 + }, + null, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [2, 2] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 12, 9, 9], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/narrow_workgroup.json b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/narrow_workgroup.json new file mode 100644 index 00000000000..2b75cb265ee --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/narrow_workgroup.json @@ -0,0 +1,242 @@ +[ + { + "name": "q8ta/conv2d/narrow_workgroup [0]", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "general", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 9, 9], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 9, 9], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/narrow_workgroup [1]", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "general", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 7, 7], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 128, 7, 7], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/performance.json b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/performance.json new file mode 100644 index 00000000000..4ac51f6ee99 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/performance.json @@ -0,0 +1,4074 @@ +[ + { + "name": "q8ta/conv2d/performance int8x4 buffer general", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "general", + "sweep_variables": { + "v_LAYOUT": ["4c1w", "4w4c", "4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 3, 256, 256], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 28], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 27 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 254, 254], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/performance int8x4 buffer auto", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "sweep_variables": { + "v_LAYOUT": ["4c1w", "4w4c", "4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 3, 256, 256], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 28], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 27 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 254, 254], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/performance [3]", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "legacy_4w4c", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 3, 256, 256], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 28], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 27 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 254, 254], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/performance int8x4 buffer 4c1w", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/performance int8x4 buffer 4c1w (4)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 576], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/performance int8x4 buffer 4c1w (5)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 512, 7, 7], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [512, 4608], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [512], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [512], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [512], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 512, 7, 7], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/performance int8x4 buffer 4w4c", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "legacy_4w4c", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/performance int8x4 buffer 4w4c (7)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "legacy_4w4c", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 576], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/performance int8x4 buffer 4w4c (8)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "legacy_4w4c", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 512, 7, 7], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [512, 4608], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [512], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [512], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [512], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 512, 7, 7], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/performance int8x4 buffer 4c", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/performance int8x4 buffer 4c (10)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 576], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/performance int8x4 buffer 4c (11)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 512, 7, 7], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [512, 4608], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [512], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [512], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [512], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 512, 7, 7], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/performance int8x4 buffer 4c1w (12)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 2 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/performance int8x4 buffer 4c1w (13)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 256, 256], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 2 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 256, 256], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/performance int8x4 buffer 4w4c (14)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "legacy_4w4c", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 2 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/performance int8x4 buffer 4w4c (15)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "legacy_4w4c", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 256, 256], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 2 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 256, 256], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/performance int8x4 buffer 4c (16)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 2 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/performance int8x4 buffer 4c (17)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 256, 256], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 2 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 256, 256], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/performance int8x4 buffer 4c1w (18)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 96, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [96, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [96], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [96], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [96], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 3 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 96, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/performance int8x4 buffer 4w4c (19)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "legacy_4w4c", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 96, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [96, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [96], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [96], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [96], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 3 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 96, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/performance int8x4 buffer 4c (20)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 96, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [96, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [96], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [96], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [96], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 3 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 96, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/performance [97]", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 3, 256, 256], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 28], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 27 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/performance [98]", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [60, 3, 256, 256], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 28], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 27 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [60, 32, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/performance [99]", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [60, 256, 10, 13], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [512, 2304], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [512], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [512], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [512], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [60, 512, 5, 7], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/performance int8x4 buffer 4c1w (24)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 800], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [5, 5] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int": 4 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 128, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/performance int8x4 buffer 4c1w (25)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128, 256, 256], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 800], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [5, 5] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int": 4 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 128, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/performance int8x4 buffer 4w4c (26)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "legacy_4w4c", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 800], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [5, 5] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int": 4 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 128, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/performance int8x4 buffer 4w4c (27)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "legacy_4w4c", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128, 256, 256], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 800], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [5, 5] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int": 4 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 128, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/performance int8x4 buffer 4c (28)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 800], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [5, 5] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int": 4 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 128, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/performance int8x4 buffer 4c (29)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128, 256, 256], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 800], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [5, 5] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int": 4 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 128, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/performance int8x4 buffer 4c1w (30)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 1024, 14, 14], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [2048, 1024], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [2048], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [2048], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [2048], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 2048, 7, 7], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/performance int8x4 buffer 4w4c (31)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "legacy_4w4c", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 1024, 14, 14], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [2048, 1024], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [2048], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [2048], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [2048], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 2048, 7, 7], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/performance int8x4 buffer 4c (32)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["general", "im2col", "auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 1024, 14, 14], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [2048, 1024], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [2048], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [2048], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [2048], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 2048, 7, 7], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/pw_correctness.json b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/pw_correctness.json new file mode 100644 index 00000000000..19b3062de7b --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/pw_correctness.json @@ -0,0 +1,3355 @@ +[ + { + "name": "q8ta/conv2d/pw_correctness", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [1, 16], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 16 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 1, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_correctness (2)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [2, 16], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 16 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 2, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_correctness (3)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [3, 16], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 16 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 3, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_correctness (4)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto", "legacy_4w4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [1, 16], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 16 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 1, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_correctness (5)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto", "legacy_4w4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [2, 16], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 16 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 2, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_correctness (6)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto", "legacy_4w4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [3, 16], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 16 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 3, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_correctness (7)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [1, 16], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 16 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 1, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_correctness (8)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [2, 16], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 16 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 2, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_correctness (9)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [3, 16], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 16 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 3, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_correctness (10)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 3, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 4], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 3 + } + }, + "valid_columns": 3 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_correctness (11)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto", "legacy_4w4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 3, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 4], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 3 + } + }, + "valid_columns": 3 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_correctness (12)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 3, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 4], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 3 + } + }, + "valid_columns": 3 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_correctness (13)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 32, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 32], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 32 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 32, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_correctness (14)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto", "legacy_4w4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 32, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 32], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 32 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 32, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_correctness (15)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 32, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 32], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 32 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 32, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_correctness (16)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [96, 64], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 64 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [96], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [96], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [96], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 96, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_correctness (17)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto", "legacy_4w4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [96, 64], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 64 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [96], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [96], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [96], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 96, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_correctness (18)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [96, 64], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 64 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [96], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [96], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [96], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 96, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_correctness (19)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 7, 57, 33], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [13, 8], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 7 + } + }, + "valid_columns": 7 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [16], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 13, 57, 33], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_correctness (20)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto", "legacy_4w4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 7, 57, 33], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [13, 8], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 7 + } + }, + "valid_columns": 7 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [16], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 13, 57, 33], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_correctness (21)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 7, 57, 33], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [13, 8], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 7 + } + }, + "valid_columns": 7 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [16], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 13, 57, 33], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_correctness (22)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 40, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [80, 40], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 40 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [80], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [80], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [80], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 80, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_correctness (23)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto", "legacy_4w4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 40, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [80, 40], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 40 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [80], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [80], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [80], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 80, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_correctness (24)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 40, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [80, 40], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 40 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [80], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [80], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [80], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 80, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_correctness (25)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto"], + "v_OUTPUT_LAYOUT": ["4c1w", "4w4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [2, 8, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [8, 8], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 8 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [8], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [8], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [8], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [2, 8, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_OUTPUT_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_correctness (26)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 7, 7, 5], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-128, -5] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": -128 + }, + { + "tensor": { + "name": "weight", + "sizes": [13, 8], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 7 + } + }, + "valid_columns": 7 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [16], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + null, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 13, 7, 5], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/pw_performance.json b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/pw_performance.json new file mode 100644 index 00000000000..62e7ec82731 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/pw_performance.json @@ -0,0 +1,2178 @@ +[ + { + "name": "q8ta/conv2d/pw_performance", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 480, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [160, 480], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 480 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [160], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [160], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [160], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 160, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_performance (2)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto", "legacy_4w4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 480, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [160, 480], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 480 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [160], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [160], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [160], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 160, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_performance (3)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 480, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [160, 480], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 480 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [160], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [160], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [160], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 160, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_performance (4)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 48, 256, 256], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [22, 48], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 48 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [24], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [24], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [24], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 22, 256, 256], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_performance (5)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto", "legacy_4w4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 48, 256, 256], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [22, 48], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 48 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [24], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [24], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [24], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 22, 256, 256], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_performance (6)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 48, 256, 256], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [22, 48], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 48 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [24], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [24], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [24], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 22, 256, 256], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_performance (7)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 48, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [48, 48], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 48 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [48], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [48], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [48], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 48, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_performance (8)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto", "legacy_4w4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 48, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [48, 48], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 48 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [48], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [48], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [48], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 48, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_performance (9)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 48, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [48, 48], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 48 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [48], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [48], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [48], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 48, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_performance (10)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 128], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 128 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 128, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_performance (11)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto", "legacy_4w4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 128], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 128 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 128, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_performance (12)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 128], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 128 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 128, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_performance (13)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 576, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 576], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 576 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_performance (14)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto", "legacy_4w4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 576, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 576], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 576 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_performance (15)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 576, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 576], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 576 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_performance (16)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 32], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 32 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/pw_performance (17)", + "operator": "et_vk.q8ta_conv2d_pw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "pw_signed", "pw_unsigned", "pw_auto"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [60, 32, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 32], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "pad_columns": { + "source": { + "rand_uniform_int": [-128, 127] + }, + "valid_columns": 32 + } + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [60, 64, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/scenex_grouped.json b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/scenex_grouped.json new file mode 100644 index 00000000000..2b8bb602687 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/scenex_grouped.json @@ -0,0 +1,371 @@ +[ + { + "name": "q8ta/conv2d/scenex_grouped", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 10, + "warmup_runs": 3, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "general", "im2col"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [60, 64, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "rand_uniform_int": [-8, 12] + } + } + }, + { + "float": 1.0 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 2 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [60, 64, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/scenex_grouped (2)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 10, + "warmup_runs": 3, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "general", "im2col"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [60, 64, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "rand_uniform_int": [-8, 12] + } + } + }, + { + "float": 1.0 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 2 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [60, 64, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/scenex_grouped (3)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 10, + "warmup_runs": 3, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "general", "im2col"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [60, 128, 128, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "rand_uniform_int": [-8, 12] + } + } + }, + { + "float": 1.0 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 800], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [5, 5] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int": 4 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [60, 128, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/scenex_regular.json b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/scenex_regular.json new file mode 100644 index 00000000000..a05e32bac5b --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/scenex_regular.json @@ -0,0 +1,248 @@ +[ + { + "name": "q8ta/conv2d/scenex_regular", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 10, + "warmup_runs": 3, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "general", "im2col"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [60, 64, 40, 51], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "rand_uniform_int": [-8, 12] + } + } + }, + { + "float": 1.0 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 576], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [60, 128, 20, 26], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/scenex_regular (2)", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 10, + "warmup_runs": 3, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0, + "rtol": 0 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "general", "im2col"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [60, 128, 20, 26], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c", + "data": { + "rand_uniform_int": [-8, 12] + } + } + }, + { + "float": 1.0 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 1152], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [256], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [256], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [256], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [60, 256, 10, 13], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/streaming_dynamic_shrink.json b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/streaming_dynamic_shrink.json new file mode 100644 index 00000000000..4b8103fa770 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/streaming_dynamic_shrink.json @@ -0,0 +1,200 @@ +{ + "name": "q8ta/conv2d/streaming_dynamic_shrink", + "check": "q8ta_streaming_dynamic_shrink", + "parameters": { + "operator_case": { + "name": "q8ta streaming dynamic shrink operator", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "im2col_auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [10, 64, 30, 99], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-8, 12] + } + } + }, + { + "float": 1.0 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [4, 576], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [ + 0.004999999888241291, 0.014999999664723873 + ] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [10, 4, 30, 99], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + "input_name": "input", + "output_name": "output", + "stream_plan": { + "batch": 10, + "flattened_kernel_size": 576, + "out_height": 30, + "out_width": 99, + "scratch_budget_bytes": 16777216 + }, + "expected_plan": { + "num_tiles": 2, + "min_rows_per_tile": 21 + }, + "resized_width": 51 + }, + "validation_steps": [ + { + "tensor_sizes": { + "input": [10, 64, 30, 99] + }, + "expected_output_sizes": [10, 4, 30, 99], + "validate_output": false + }, + { + "tensor_sizes": { + "input": [1, 64, 20, 51] + }, + "expected_output_sizes": [1, 4, 20, 51], + "validate_output": true + }, + { + "tensor_sizes": { + "input": [10, 64, 30, 99] + }, + "expected_output_sizes": [10, 4, 30, 99], + "validate_output": false + }, + { + "tensor_sizes": { + "input": [1, 64, 20, 51] + }, + "expected_output_sizes": [1, 4, 20, 51], + "validate_output": true + }, + { + "tensor_sizes": { + "input": [10, 64, 30, 99] + }, + "expected_output_sizes": [10, 4, 30, 99], + "validate_output": false + }, + { + "tensor_sizes": { + "input": [1, 64, 20, 51] + }, + "expected_output_sizes": [1, 4, 20, 51], + "validate_output": true + }, + { + "tensor_sizes": { + "input": [10, 64, 30, 99] + }, + "expected_output_sizes": [10, 4, 30, 99], + "validate_output": false + }, + { + "tensor_sizes": { + "input": [1, 64, 20, 51] + }, + "expected_output_sizes": [1, 4, 20, 51], + "validate_output": true + } + ] +} diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/streaming_im2col.json b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/streaming_im2col.json new file mode 100644 index 00000000000..1097d2d7015 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/streaming_im2col.json @@ -0,0 +1,725 @@ +[ + { + "name": "q8ta/conv2d/streaming_im2col [0]", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "im2col_fallback", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [10, 32, 30, 99], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-8, 12] + } + } + }, + { + "float": 1.0 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [4, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [10, 4, 30, 99], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/streaming_im2col [1]", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "im2col_fallback", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [10, 64, 30, 99], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-8, 12] + } + } + }, + { + "float": 1.0 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [4, 576], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [10, 4, 30, 99], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/streaming_im2col", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "im2col_auto", + "sweep_variables": { + "v_LAYOUT": ["4c1w", "4w4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [10, 32, 30, 99], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-8, 12] + } + } + }, + { + "float": 1.0 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [4, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [10, 4, 30, 99], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/streaming_im2col [4]", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "im2col_auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [10, 64, 30, 99], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "rand_uniform_int": [-8, 12] + } + } + }, + { + "float": 1.0 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [4, 576], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [10, 4, 30, 99], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4c1w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/streaming_im2col [6]", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "im2col_auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [20, 32, 30, 99], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-8, 12] + } + } + }, + { + "float": 1.0 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [10, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [12], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [12], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [12], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [20, 10, 30, 99], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d/streaming_im2col [5]", + "operator": "et_vk.q8ta_conv2d.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "im2col_auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [20, 32, 30, 99], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-8, 12] + } + } + }, + { + "float": 1.0 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [16, 144], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [16], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 2 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [20, 16, 30, 99], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/unsigned_dot_selection.json b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/unsigned_dot_selection.json new file mode 100644 index 00000000000..9ea92c4d2a1 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/cases/unsigned_dot_selection.json @@ -0,0 +1,8 @@ +{ + "name": "q8ta/conv2d/unsigned_dot_selection", + "check": "q8ta_unsigned_dot_selection", + "parameters": { + "accumulator_limit_bytes": 33025, + "test_kernel_sizes": [33025, 33026] + } +} diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/test/q8ta_native_reference_test.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/test/q8ta_native_reference_test.cpp new file mode 100644 index 00000000000..cc60107bba8 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/test/q8ta_native_reference_test.cpp @@ -0,0 +1,159 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include + +#include + +#include + +namespace vkcompute::shader_profiling { +namespace { +using reference::Tensor; +using reference::TensorMap; + +Json native_spec() { + return { + {"operator", "et_vk.q8ta_conv2d.default"}, + {"args", + {{{"tensor", "x"}}, + {{"float", 1.0}}, + {{"int", 0}}, + {{"tensor", "w"}}, + {{"tensor", "sums"}}, + {{"tensor", "scales"}}, + {{"float", 1.0}}, + {{"int", 0}}, + {{"tensor", "bias"}}, + {{"int_list", {1, 1}}}, + {{"int_list", {1, 1}}}, + {{"int_list", {0, 0}}}, + {{"int_list", {1, 1}}}, + {{"int", 1}}, + {{"string", "none"}}, + {{"tensor", "y"}}}}, + {"tensors", + {{"y", + {{"sizes", {1, 1, 1, 1}}, + {"dtype", "int8x4"}, + {"storage", "buffer"}, + {"layout", "4c1w"}}}}}}; +} + +TEST(Q8taNativeReference, RegistersNativeOperatorsOnly) { + for (const char* name : + {"et_vk.q8ta_conv2d.default", + "et_vk.q8ta_conv2d_general.default", + "et_vk.q8ta_conv2d_im2col.default", + "et_vk.q8ta_conv2d_dw.default", + "et_vk.q8ta_conv2d_pw.default"}) + EXPECT_EQ(reference_registry().operator_reference(name), "q8ta_conv2d"); + EXPECT_EQ( + reference_registry().operator_reference( + "et_vk.q8ta_conv2d_transposed.default"), + "q8ta_transposed"); + EXPECT_THROW( + reference_registry().operator_reference( + "benchmark_v2.q8ta_conv2d.default"), + std::invalid_argument); +} + +TEST( + Q8taNativeReference, + QuantizedInputOutputHonorZeroPointsBiasAndPaddedRows) { + Json spec = native_spec(); + spec["args"][1]["float"] = .5; + spec["args"][2]["int"] = 2; + spec["args"][6]["float"] = .5; + spec["args"][7]["int"] = -3; + const TensorMap data{ + {"x", {{1, 1, 1, 1}, {5}}}, + {"w", {{1, 4}, {2, 0, 0, 0}}}, + {"sums", {{1}, {2}}}, + {"scales", {{1}, {.25}}}, + {"bias", {{1}, {.25}}}}; + EXPECT_EQ( + reference::q8ta_conv2d_reference(spec, data, 1).values, + (std::vector{-1})); +} + +TEST(Q8taNativeReference, DepthwiseHwcWeightsPreserveChannelAndKernelOrder) { + Json spec = native_spec(); + spec["operator"] = "et_vk.q8ta_conv2d_dw.default"; + spec["args"][8] = nullptr; + spec["args"][9]["int_list"] = {1, 2}; + spec["args"][13]["int"] = 2; + spec["tensors"]["y"]["sizes"] = {1, 2, 1, 2}; + const TensorMap data{ + {"x", {{1, 2, 1, 3}, {1, 2, 3, 10, 20, 30}}}, + {"w", {{1, 2, 2}, {1, 4, 2, -1}}}, + {"sums", {{2}, {3, 3}}}, + {"scales", {{2}, {1, 1}}}}; + const auto one = reference::q8ta_conv2d_reference(spec, data, 1); + EXPECT_EQ(one.values, (std::vector{5, 8, 20, 50})); + EXPECT_EQ(reference::q8ta_conv2d_reference(spec, data, 4).values, one.values); +} + +TEST(Q8taNativeReference, TransposedOutputPaddingUsesNativeArgumentPositions) { + Json spec = native_spec(); + spec["operator"] = "et_vk.q8ta_conv2d_transposed.default"; + spec["args"][8] = nullptr; + spec["args"][9]["int_list"] = {1, 2}; + spec["args"][10]["int_list"] = {1, 2}; + spec["args"].insert(spec["args"].begin() + 12, Json{{"int_list", {0, 1}}}); + spec["tensors"]["y"]["sizes"] = {1, 1, 1, 5}; + const TensorMap data{ + {"x", {{1, 1, 1, 2}, {1, 2}}}, + {"w", {{1, 4}, {1, 2, 0, 0}}}, + {"sums", {{1}, {3}}}, + {"scales", {{1}, {1}}}}; + EXPECT_EQ( + reference::q8ta_transposed_reference(spec, data, 2).values, + (std::vector{1, 2, 2, 4, 0})); +} + +TEST(Q8taNativeReference, RequantizesNearestEvenAndSaturatesWithRelu) { + Json spec = native_spec(); + spec["args"][1]["float"] = .5; + spec["args"][8] = nullptr; + spec["tensors"]["y"]["sizes"] = {1, 1, 1, 6}; + const TensorMap data{ + {"x", {{1, 1, 1, 6}, {1, 3, -1, -3, 127, -128}}}, + {"w", {{1, 4}, {1, 0, 0, 0}}}, + {"sums", {{1}, {1}}}, + {"scales", {{1}, {1}}}}; + EXPECT_EQ( + reference::q8ta_conv2d_reference(spec, data, 1).values, + (std::vector{0, 2, 0, -2, 64, -64})); + spec["args"][1]["float"] = 4; + spec["args"][14]["string"] = "relu"; + EXPECT_EQ( + reference::q8ta_conv2d_reference(spec, data, 1).values, + (std::vector{4, 12, 0, 0, 127, 0})); +} + +TEST(Q8taNativeReference, ActualShaderSelectsIntegerCorrectionAndFusedBias) { + Json spec = native_spec(); + Tensor input{{1, 1042, 1, 1}, std::vector(1042, 127)}; + Tensor weight{{1, 1044}, std::vector(1044, 127)}; + input.values[1041] = 1; + weight.values[1040] = 24; + weight.values[1041] = 7; + weight.values[1042] = weight.values[1043] = 0; + const TensorMap data{ + {"x", input}, + {"w", weight}, + {"sums", {{1}, {132111}}}, + {"scales", {{1}, {std::nextafter(1.0f, 2.0f)}}}, + {"bias", {{1}, {-16777216}}}}; + spec["reference_context"]["shaders"] = {"q8ta_conv2d_float"}; + EXPECT_EQ(reference::q8ta_conv2d_reference(spec, data, 1).values[0], 0); + spec["reference_context"]["shaders"] = {"q8ta_conv2d_pw_float"}; + EXPECT_EQ(reference::q8ta_conv2d_reference(spec, data, 1).values[0], 1); +} +} // namespace +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/transposed/Q8taTransposedReference.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/transposed/Q8taTransposedReference.cpp new file mode 100644 index 00000000000..22dd82fc7e7 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/transposed/Q8taTransposedReference.cpp @@ -0,0 +1,115 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include +#include +#include + +namespace vkcompute::shader_profiling::reference { +Tensor q8ta_transposed_reference( + const Json& spec, + const TensorMap& tensors, + size_t threads) { + const Tensor& input = tensor_arg(spec, 0, tensors); + const Tensor& weight = tensor_arg(spec, 3, tensors); + const Tensor& sums = tensor_arg(spec, 4, tensors); + const Tensor& scales = tensor_arg(spec, 5, tensors); + const Tensor* bias = + spec.at("args").at(8).is_null() ? nullptr : &tensor_arg(spec, 8, tensors); + Tensor out = q8ta::output_tensor(spec, 16); + const auto kernel = conv2d_pair_arg(spec, 9); + const auto stride = conv2d_pair_arg(spec, 10); + const auto pad = conv2d_pair_arg(spec, 11); + const auto output_padding = conv2d_pair_arg(spec, 12); + const auto dilation = conv2d_pair_arg(spec, 13); + const int64_t groups = q8ta::integer(spec, 14); + require( + input.sizes.size() == 4 && out.sizes.size() == 4 && groups > 0, + "Q8 transposed convolution requires NCHW tensors and positive groups"); + const int64_t cin = input.sizes[1], cout = out.sizes[1]; + const int64_t height = out.sizes[2], width = out.sizes[3]; + require( + cin % groups == 0 && cout % groups == 0, + "Q8 transposed convolution channels must be divisible by groups"); + const int64_t columns = cin / groups * kernel[0] * kernel[1]; + require( + weight.sizes.size() == 2 && weight.sizes[0] == cout && + weight.sizes[1] >= columns && sums.values.size() >= size_t(cout) && + scales.values.size() >= size_t(cout) && + (!bias || bias->values.size() >= size_t(cout)), + "Q8 transposed convolution requires OHWI weights and channel parameters"); + for (size_t d = 0; d < 2; ++d) { + require( + kernel[d] > 0 && stride[d] > 0 && dilation[d] > 0 && pad[d] >= 0 && + output_padding[d] >= 0 && + (output_padding[d] < stride[d] || output_padding[d] < dilation[d]), + "Q8 transposed convolution spatial parameters are invalid"); + require( + out.sizes[d + 2] == + (input.sizes[d + 2] - 1) * stride[d] - 2 * pad[d] + + dilation[d] * (kernel[d] - 1) + output_padding[d] + 1, + "Q8 transposed convolution output shape disagrees with parameters"); + } + require(out.sizes[0] == input.sizes[0], "Q8 transposed batch sizes disagree"); + const float is = q8ta::scale(spec, 1), os = q8ta::scale(spec, 6); + const int64_t iz = q8ta::integer(spec, 2), oz = q8ta::integer(spec, 7); + require( + std::isfinite(is) && is > 0 && std::isfinite(os) && os > 0 && + iz >= -128 && iz <= 127 && oz >= -128 && oz <= 127, + "Q8 transposed convolution quantization parameters are invalid"); + const std::string activation = spec.at("args").at(15).at("string"); + require( + activation == "none" || activation == "relu", + "Q8 transposed convolution reference supports none/relu"); + std::vector logical_sums(size_t(cout), 0); + for (int64_t oc = 0; oc < cout; ++oc) + for (int64_t k = 0; k < columns; ++k) + logical_sums[size_t(oc)] += + int64_t(weight.values[size_t(oc * weight.sizes[1] + k)]); + parallel_for(out.values.size(), threads, [&](size_t index) { + const int64_t x = int64_t(index) % width; + const int64_t y = int64_t(index) / width % height; + const int64_t oc = int64_t(index) / (width * height) % cout; + const int64_t n = int64_t(index) / (width * height * cout); + const int64_t group = oc / (cout / groups); + int64_t sum = 0; + for (int64_t c = 0; c < cin / groups; ++c) + for (int64_t ky = 0; ky < kernel[0]; ++ky) + for (int64_t kx = 0; kx < kernel[1]; ++kx) { + const int64_t iy = y + pad[0] - ky * dilation[0]; + const int64_t ix = x + pad[1] - kx * dilation[1]; + if (iy < 0 || ix < 0 || iy % stride[0] || ix % stride[1] || + iy / stride[0] >= input.sizes[2] || + ix / stride[1] >= input.sizes[3]) + continue; + const int64_t source = + ((n * cin + group * (cin / groups) + c) * input.sizes[2] + + iy / stride[0]) * + input.sizes[3] + + ix / stride[1]; + const int64_t wi = + oc * weight.sizes[1] + (ky * kernel[1] + kx) * (cin / groups) + c; + sum += (int64_t(input.values[size_t(source)]) - iz) * + int64_t(weight.values[size_t(wi)]); + } + const int64_t raw = sum + iz * logical_sums[size_t(oc)]; + const float corrected = + float(raw) - float(sums.values[size_t(oc)]) * float(iz); + float value = corrected * (float(scales.values[size_t(oc)]) * is); + value += bias ? float(bias->values[size_t(oc)]) : 0; + if (activation == "relu") + value = std::max(value, 0.0f); + out.values[index] = double(q8ta::requantize(value, 1.0f / os, oz)); + }); + return out; +} +REGISTER_REFERENCE_FUNCTIONS { + REGISTER_REFERENCE_FN(q8ta_transposed, q8ta_transposed_reference); + REGISTER_OPERATOR_REFERENCE( + et_vk.q8ta_conv2d_transposed.default, q8ta_transposed); +} +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/transposed/cases/correctness.json b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/transposed/cases/correctness.json new file mode 100644 index 00000000000..d683c61ba61 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/transposed/cases/correctness.json @@ -0,0 +1,627 @@ +[ + { + "name": "q8ta/conv2d_transposed/correctness", + "operator": "et_vk.q8ta_conv2d_transposed.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4c1w", "4w4c", "4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [16, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [16], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d_transposed/correctness (2)", + "operator": "et_vk.q8ta_conv2d_transposed.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4c1w", "4w4c", "4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 4, 4], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 576], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d_transposed/correctness (3)", + "operator": "et_vk.q8ta_conv2d_transposed.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4c1w", "4w4c", "4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 2 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d_transposed/correctness (4)", + "operator": "et_vk.q8ta_conv2d_transposed.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4c1w", "4w4c", "4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [16, 512], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [16], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [4, 4] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d_transposed/correctness (5)", + "operator": "et_vk.q8ta_conv2d_transposed.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4c1w", "4w4c", "4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [16, 144], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [16], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/transposed/cases/debug.json b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/transposed/cases/debug.json new file mode 100644 index 00000000000..011df2c097c --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/transposed/cases/debug.json @@ -0,0 +1,127 @@ +[ + { + "name": "q8ta/conv2d_transposed/debug", + "operator": "et_vk.q8ta_conv2d_transposed.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4c1w", "4w4c", "4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [16, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [16], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/transposed/cases/performance.json b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/transposed/cases/performance.json new file mode 100644 index 00000000000..42421212493 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/q8ta/transposed/cases/performance.json @@ -0,0 +1,377 @@ +[ + { + "name": "q8ta/conv2d_transposed/performance", + "operator": "et_vk.q8ta_conv2d_transposed.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4c1w", "4w4c", "4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 2048], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [4, 4] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 32, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d_transposed/performance (2)", + "operator": "et_vk.q8ta_conv2d_transposed.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4c1w", "4w4c", "4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 256, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 4096], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [4, 4] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 128, 32, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/conv2d_transposed/performance (3)", + "operator": "et_vk.q8ta_conv2d_transposed.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4c1w", "4w4c", "4c"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128, 32, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-121, 125] + } + } + }, + { + "float": 0.008123000152409077 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 1152], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 64, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/qw/q8csw/Q8cswConv2dReference.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/qw/q8csw/Q8cswConv2dReference.cpp new file mode 100644 index 00000000000..25321801aba --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/qw/q8csw/Q8cswConv2dReference.cpp @@ -0,0 +1,78 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include +#include +#include +namespace vkcompute::shader_profiling::reference { +static Tensor q8csw_conv2d_reference( + const Json& spec, + const TensorMap& tensors, + size_t threads) { + const bool quantized = + spec.at("operator") == "et_vk.conv2d_q8ta_q8csw.default"; + const size_t wi = quantized ? 3 : 1, si = quantized ? 5 : 2, + bi = quantized ? 6 : 3, ki = quantized ? 7 : 4; + Tensor input = tensor_arg(spec, 0, tensors); + const Tensor& weight = tensor_arg(spec, wi, tensors); + const Tensor& scales = tensor_arg(spec, si, tensors); + const Tensor* bias = spec.at("args").at(bi).is_null() + ? nullptr + : &tensor_arg(spec, bi, tensors); + const Conv2dParameters p{ + conv2d_pair_arg(spec, ki), + conv2d_pair_arg(spec, ki + 1), + conv2d_pair_arg(spec, ki + 2), + conv2d_pair_arg(spec, ki + 3), + q8ta::integer(spec, ki + 4)}; + const auto g = conv2d_geometry(input, weight, bias, p); + const auto weight_index = + [&](int64_t oc, int64_t ci, int64_t ky, int64_t kx) { + return oc * weight.sizes[1] + + (ky * p.kernel[1] + kx) * g.in_channels_per_group + ci; + }; + if (quantized) { + const float scale = q8ta::scale(spec, 1); + const int64_t zp = q8ta::integer(spec, 2); + input = q8ta::quantized_input(input, scale, zp); + return convolve( + input, + weight, + p, + g, + threads, + weight_index, + [&](int64_t sum, double value, double w) { + return sum + (int64_t(value) - zp) * int64_t(w); + }, + [&](int64_t sum, int64_t oc) { + const float combined_scale = scale * float(scales.values[size_t(oc)]); + return std::fma(float(sum), combined_scale, 0.0f) + + (bias ? float(bias->values[size_t(oc)]) : 0.0f); + }); + } + return convolve( + input, + weight, + p, + g, + threads, + weight_index, + [](float sum, double value, double w) { + return sum + float(value) * float(w); + }, + [&](float sum, int64_t oc) { + return sum * float(scales.values[size_t(oc)]) + + (bias ? float(bias->values[size_t(oc)]) : 0.0f); + }); +} +REGISTER_REFERENCE_FUNCTIONS { + REGISTER_REFERENCE_FN(q8csw_conv2d, q8csw_conv2d_reference); + REGISTER_OPERATOR_REFERENCE(et_vk.conv2d_q8csw.default, q8csw_conv2d); + REGISTER_OPERATOR_REFERENCE(et_vk.conv2d_q8ta_q8csw.default, q8csw_conv2d); +} +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/qw/q8csw/cases/correctness.json b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/qw/q8csw/cases/correctness.json new file mode 100644 index 00000000000..85198f03ba4 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/qw/q8csw/cases/correctness.json @@ -0,0 +1,2396 @@ +[ + { + "name": "q8csw_conv2d/correctness", + "operator": "et_vk.conv2d_q8ta_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 3, 64, 64], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 0.07000000029802322 + }, + { + "int": -3 + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 28], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 32, 32], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 3, 64, 64], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 0.07000000029802322 + }, + { + "int": -3 + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 28], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 32, 32], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_conv2d/correctness [10]", + "operator": "et_vk.conv2d_q8ta_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 32, 32], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 0.07000000029802322 + }, + { + "int": -3 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 16, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_conv2d/correctness [2]", + "operator": "et_vk.conv2d_q8ta_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 32, 32], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 0.07000000029802322 + }, + { + "int": -3 + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 144], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 32, 32], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_conv2d/correctness [4]", + "operator": "et_vk.conv2d_q8ta_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 16, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 0.07000000029802322 + }, + { + "int": -3 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 16, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_conv2d/correctness [6]", + "operator": "et_vk.conv2d_q8ta_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 55, 55], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 0.07000000029802322 + }, + { + "int": -3 + }, + { + "tensor": { + "name": "weight", + "sizes": [1, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [1], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [1], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [1], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 1, 55, 55], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_conv2d/correctness [12]", + "operator": "et_vk.conv2d_q8ta_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 28, 28], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 0.07000000029802322 + }, + { + "int": -3 + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 400], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [5, 5] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 28, 28], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_conv2d/correctness [14]", + "operator": "et_vk.conv2d_q8ta_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 14, 14], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 0.07000000029802322 + }, + { + "int": -3 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 1568], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [7, 7] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 14, 14], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_conv2d/correctness [20]", + "operator": "et_vk.conv2d_q8ta_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 32, 32], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 0.07000000029802322 + }, + { + "int": -3 + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 72], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 4 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 32, 32], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_conv2d/correctness [22]", + "operator": "et_vk.conv2d_q8ta_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 16, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 0.07000000029802322 + }, + { + "int": -3 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 72], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 8 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 16, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_conv2d/correctness [16]", + "operator": "et_vk.conv2d_q8ta_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 32, 32], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 0.07000000029802322 + }, + { + "int": -3 + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 144], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [2, 2] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 32, 32], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_conv2d/correctness [18]", + "operator": "et_vk.conv2d_q8ta_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 16, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 0.07000000029802322 + }, + { + "int": -3 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [3, 3] + }, + { + "int_list": [3, 3] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 16, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_conv2d/correctness (12)", + "operator": "et_vk.conv2d_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 3, 64, 64], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 28], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 32, 32], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 3, 64, 64], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 28], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 32, 32], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_conv2d/correctness [11]", + "operator": "et_vk.conv2d_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 32, 32], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 16, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_conv2d/correctness [3]", + "operator": "et_vk.conv2d_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 32, 32], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 144], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 32, 32], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_conv2d/correctness [5]", + "operator": "et_vk.conv2d_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 16, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 16, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_conv2d/correctness [7]", + "operator": "et_vk.conv2d_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 55, 55], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [1, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [1], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [1], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 1, 55, 55], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_conv2d/correctness [21]", + "operator": "et_vk.conv2d_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 32, 32], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 72], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 4 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 32, 32], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_conv2d/correctness [23]", + "operator": "et_vk.conv2d_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 16, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 72], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 8 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 16, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_conv2d/correctness [17]", + "operator": "et_vk.conv2d_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 32, 32], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 144], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [2, 2] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 32, 32], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_conv2d/correctness [19]", + "operator": "et_vk.conv2d_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 16, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 288], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [3, 3] + }, + { + "int_list": [3, 3] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 16, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_conv2d/correctness [13]", + "operator": "et_vk.conv2d_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 28, 28], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 400], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [5, 5] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [2, 2] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32, 28, 28], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_conv2d/correctness [15]", + "operator": "et_vk.conv2d_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 14, 14], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 1568], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [7, 7] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 14, 14], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/convolution/qw/q8csw/cases/performance.json b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/qw/q8csw/cases/performance.json new file mode 100644 index 00000000000..0b17381a26e --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/convolution/qw/q8csw/cases/performance.json @@ -0,0 +1,608 @@ +[ + { + "name": "q8csw_conv2d/performance [0]", + "operator": "et_vk.conv2d_q8ta_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128, 128, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 0.07000000029802322 + }, + { + "int": -3 + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 128], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [256], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [256], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [256], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 256, 130, 130], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_conv2d/performance [1]", + "operator": "et_vk.conv2d_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128, 128, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 128], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [256], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [256], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 256, 130, 130], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_conv2d/performance [2]", + "operator": "et_vk.conv2d_q8ta_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 128, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 0.07000000029802322 + }, + { + "int": -3 + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 576], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 128, 128, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_conv2d/performance [3]", + "operator": "et_vk.conv2d_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 128, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 576], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [3, 3] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 128, 128, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_conv2d/performance [4]", + "operator": "et_vk.conv2d_q8ta_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 1024, 128, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 0.07000000029802322 + }, + { + "int": -3 + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 1024], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 128, 128, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_conv2d/performance [5]", + "operator": "et_vk.conv2d_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 1024, 128, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 1024], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int_list": [1, 1] + }, + { + "int_list": [1, 1] + }, + { + "int_list": [0, 0] + }, + { + "int_list": [1, 1] + }, + { + "int": 1 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 128, 128, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/embedding/qw/q4gsw/EmbeddingReference.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/embedding/qw/q4gsw/EmbeddingReference.cpp new file mode 100644 index 00000000000..21c1dc4f51a --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/embedding/qw/q4gsw/EmbeddingReference.cpp @@ -0,0 +1,45 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include +namespace vkcompute::shader_profiling::reference { +Tensor embedding_q4gsw_reference( + const Json& spec, + const TensorMap& tensors, + size_t threads) { + const Tensor& weight = tensor_arg(spec, 0, tensors); + const Tensor& scales = tensor_arg(spec, 1, tensors); + const Tensor& indices = tensor_arg(spec, 3, tensors); + const int64_t group = spec.at("args").at(2).at("int"); + const bool linear = spec.at("args").at(4).at("bool"); + require( + weight.sizes.size() == 2 && group > 0, + "Embedding needs rank2 weight and positive group size"); + const size_t dim = weight.sizes[1] * 2, vocab = weight.sizes[0], + groups = dim / group; + require( + dim % group == 0 && scales.values.size() == vocab * groups, + "Embedding scale dimensions differ"); + Tensor out{indices.sizes, std::vector(indices.values.size() * dim)}; + out.sizes.push_back(dim); + parallel_for(indices.values.size(), threads, [&](size_t i) { + const int64_t row = indices.values[i]; + require(row >= 0 && row < int64_t(vocab), "Embedding index out of bounds"); + for (size_t d = 0; d < dim; ++d) { + const uint8_t byte = weight.values[row * (dim / 2) + d / 2]; + const size_t shift = linear ? 4 * (d % 2) : 4 * (1 - d % 2); + out.values[i * dim + d] = float(int((byte >> shift) & 15) - 8) * + float(scales.values[row * groups + d / group]); + } + }); + return out; +} +REGISTER_REFERENCE_FUNCTIONS { + REGISTER_REFERENCE_FN(embedding_q4gsw, embedding_q4gsw_reference); + REGISTER_OPERATOR_REFERENCE(et_vk.embedding_q4gsw.default, embedding_q4gsw); +} +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/embedding/qw/q4gsw/EmbeddingReference.h b/backends/vulkan/test/ops/benchmark_v2/operators/embedding/qw/q4gsw/EmbeddingReference.h new file mode 100644 index 00000000000..0af55a1f32e --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/embedding/qw/q4gsw/EmbeddingReference.h @@ -0,0 +1,18 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include + +namespace vkcompute::shader_profiling::reference { + +Tensor embedding_q4gsw_reference( + const Json& spec, + const TensorMap& tensors, + size_t threads); + +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/embedding/qw/q4gsw/cases/correctness.json b/backends/vulkan/test/ops/benchmark_v2/operators/embedding/qw/q4gsw/cases/correctness.json new file mode 100644 index 00000000000..2e90ae24b73 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/embedding/qw/q4gsw/cases/correctness.json @@ -0,0 +1,922 @@ +[ + { + "name": "embedding_q4gsw correctness", + "operator": "et_vk.embedding_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_OUTPUT_DTYPE": ["float16", "float32"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "weight", + "sizes": [16, 16], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [16, 1], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + { + "tensor": { + "name": "indices", + "sizes": [4], + "dtype": "int32", + "storage": "buffer", + "layout": "width", + "data": { + "abs_mod": { + "bounds": [-10, 10], + "mod": 16 + } + } + } + }, + { + "bool": true + }, + { + "tensor": { + "name": "output", + "sizes": [4, 32], + "dtype": "v_OUTPUT_DTYPE", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "embedding_q4gsw correctness small_1d_texture_linear_weight texture3d", + "operator": "et_vk.embedding_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "weight", + "sizes": [16, 16], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [16, 1], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + { + "tensor": { + "name": "indices", + "sizes": [4], + "dtype": "int32", + "storage": "texture3d", + "layout": "width", + "data": { + "abs_mod": { + "bounds": [-10, 10], + "mod": 16 + } + } + } + }, + { + "bool": true + }, + { + "tensor": { + "name": "output", + "sizes": [4, 32], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "embedding_q4gsw correctness small_2d_linear_weight buffer", + "operator": "et_vk.embedding_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "weight", + "sizes": [32, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [32, 2], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + { + "tensor": { + "name": "indices", + "sizes": [2, 3], + "dtype": "int32", + "storage": "buffer", + "layout": "width", + "data": { + "abs_mod": { + "bounds": [-10, 10], + "mod": 32 + } + } + } + }, + { + "bool": true + }, + { + "tensor": { + "name": "output", + "sizes": [2, 3, 64], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "embedding_q4gsw correctness medium_multigroup_linear_weight buffer", + "operator": "et_vk.embedding_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "weight", + "sizes": [100, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [100, 4], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + { + "tensor": { + "name": "indices", + "sizes": [4, 8], + "dtype": "int32", + "storage": "buffer", + "layout": "width", + "data": { + "abs_mod": { + "bounds": [-10, 10], + "mod": 100 + } + } + } + }, + { + "bool": true + }, + { + "tensor": { + "name": "output", + "sizes": [4, 8, 128], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "embedding_q4gsw correctness (5)", + "operator": "et_vk.embedding_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_SCALES_DTYPE": ["float16", "float32"], + "v_STORAGE": ["buffer", "texture3d"], + "v_OUTPUT_DTYPE": ["float16", "float32"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "weight", + "sizes": [16, 16], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [16, 1], + "dtype": "v_SCALES_DTYPE", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + { + "tensor": { + "name": "indices", + "sizes": [4], + "dtype": "int32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "abs_mod": { + "bounds": [-10, 10], + "mod": 16 + } + } + } + }, + { + "bool": false + }, + { + "tensor": { + "name": "output", + "sizes": [4, 32], + "dtype": "v_OUTPUT_DTYPE", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "embedding_q4gsw correctness (6)", + "operator": "et_vk.embedding_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["buffer", "texture3d"], + "v_OUTPUT_DTYPE": ["float16", "float32"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "weight", + "sizes": [32, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [32, 2], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + { + "tensor": { + "name": "indices", + "sizes": [2, 3], + "dtype": "int32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "abs_mod": { + "bounds": [-10, 10], + "mod": 32 + } + } + } + }, + { + "bool": false + }, + { + "tensor": { + "name": "output", + "sizes": [2, 3, 64], + "dtype": "v_OUTPUT_DTYPE", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "embedding_q4gsw correctness (7)", + "operator": "et_vk.embedding_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_OUTPUT_DTYPE": ["float16", "float32"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "weight", + "sizes": [32, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [32, 2], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + { + "tensor": { + "name": "indices", + "sizes": [2, 3], + "dtype": "int32", + "storage": "buffer", + "layout": "width", + "data": { + "abs_mod": { + "bounds": [-10, 10], + "mod": 32 + } + } + } + }, + { + "bool": false + }, + { + "tensor": { + "name": "output", + "sizes": [2, 3, 64], + "dtype": "v_OUTPUT_DTYPE", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "embedding_q4gsw correctness (8)", + "operator": "et_vk.embedding_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["buffer", "texture3d"], + "v_OUTPUT_DTYPE": ["float16", "float32"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "weight", + "sizes": [100, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [100, 4], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + { + "tensor": { + "name": "indices", + "sizes": [4, 8], + "dtype": "int32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "abs_mod": { + "bounds": [-10, 10], + "mod": 100 + } + } + } + }, + { + "bool": false + }, + { + "tensor": { + "name": "output", + "sizes": [4, 8, 128], + "dtype": "v_OUTPUT_DTYPE", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "embedding_q4gsw correctness (9)", + "operator": "et_vk.embedding_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_OUTPUT_DTYPE": ["float16", "float32"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "weight", + "sizes": [100, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [100, 4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + { + "tensor": { + "name": "indices", + "sizes": [4, 8], + "dtype": "int32", + "storage": "buffer", + "layout": "width", + "data": { + "abs_mod": { + "bounds": [-10, 10], + "mod": 100 + } + } + } + }, + { + "bool": false + }, + { + "tensor": { + "name": "output", + "sizes": [4, 8, 128], + "dtype": "v_OUTPUT_DTYPE", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "embedding_q4gsw correctness (10)", + "operator": "et_vk.embedding_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["buffer", "texture3d"], + "v_OUTPUT_DTYPE": ["float16", "float32"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "weight", + "sizes": [50, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [50, 1], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 64 + }, + { + "tensor": { + "name": "indices", + "sizes": [2, 4], + "dtype": "int32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "abs_mod": { + "bounds": [-10, 10], + "mod": 50 + } + } + } + }, + { + "bool": false + }, + { + "tensor": { + "name": "output", + "sizes": [2, 4, 64], + "dtype": "v_OUTPUT_DTYPE", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "embedding_q4gsw correctness gemma4_token_embedding_per_row", + "operator": "et_vk.embedding_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["buffer", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "weight", + "sizes": [16, 768], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 1536 + }, + { + "tensor": { + "name": "indices", + "sizes": [1, 4], + "dtype": "int32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "abs_mod": { + "bounds": [-10, 10], + "mod": 16 + } + } + } + }, + { + "bool": true + }, + { + "tensor": { + "name": "output", + "sizes": [1, 4, 1536], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "embedding_q4gsw correctness gemma4_ple_embedding_per_row", + "operator": "et_vk.embedding_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["buffer", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "weight", + "sizes": [16, 4480], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 8960 + }, + { + "tensor": { + "name": "indices", + "sizes": [1, 4], + "dtype": "int32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "abs_mod": { + "bounds": [-10, 10], + "mod": 16 + } + } + } + }, + { + "bool": true + }, + { + "tensor": { + "name": "output", + "sizes": [1, 4, 8960], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/embedding/qw/q4gsw/cases/performance.json b/backends/vulkan/test/ops/benchmark_v2/operators/embedding/qw/q4gsw/cases/performance.json new file mode 100644 index 00000000000..8e93c4808d8 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/embedding/qw/q4gsw/cases/performance.json @@ -0,0 +1,154 @@ +[ + { + "name": "embedding_q4gsw performance llama_3_2_1b_prefill_fp32_linear_weight buffer", + "operator": "et_vk.embedding_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "weight", + "sizes": [128256, 1024], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [128256, 64], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + { + "tensor": { + "name": "indices", + "sizes": [1, 2047], + "dtype": "int32", + "storage": "buffer", + "layout": "width", + "data": { + "abs_mod": { + "bounds": [-10, 10], + "mod": 128256 + } + } + } + }, + { + "bool": true + }, + { + "tensor": { + "name": "output", + "sizes": [1, 2047, 2048], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "embedding_q4gsw performance", + "operator": "et_vk.embedding_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_SCALES_DTYPE": ["float16", "float32"], + "v_OUTPUT_DTYPE": ["float32", "float16"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "weight", + "sizes": [128256, 1024], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [128256, 64], + "dtype": "v_SCALES_DTYPE", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + { + "tensor": { + "name": "indices", + "sizes": [1, 2047], + "dtype": "int32", + "storage": "buffer", + "layout": "width", + "data": { + "abs_mod": { + "bounds": [-10, 10], + "mod": 128256 + } + } + } + }, + { + "bool": false + }, + { + "tensor": { + "name": "output", + "sizes": [1, 2047, 2048], + "dtype": "v_OUTPUT_DTYPE", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/GemmReference.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/GemmReference.cpp new file mode 100644 index 00000000000..e6f22f684c0 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/GemmReference.cpp @@ -0,0 +1,68 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include + +namespace vkcompute::shader_profiling::reference { + +Tensor linear_projection( + const Tensor& input, + const Tensor& weight, + const Tensor* bias, + size_t threads) { + require( + !input.sizes.empty() && weight.sizes.size() == 2, + "Linear requires input rank >= 1 and weight rank 2"); + const size_t k = input.sizes.back(); + const size_t n = weight.sizes[0]; + require( + k == size_t(weight.sizes[1]), "Linear input/weight dimensions differ"); + require( + !bias || bias->values.size() == n, + "Linear bias size must equal output features"); + const size_t m = input.values.size() / k; + Tensor output{input.sizes, std::vector(m * n)}; + output.sizes.back() = n; + std::vector packed(k * n); + parallel_for(n, threads, [&](size_t column) { + for (size_t inner = 0; inner < k; ++inner) { + packed[inner * n + column] = weight.values[column * k + inner]; + } + }); + std::vector activations(input.values.begin(), input.values.end()); + constexpr size_t kRows = 4; + constexpr size_t kColumns = 64; + const size_t column_tiles = (n + kColumns - 1) / kColumns; + const size_t row_tiles = (m + kRows - 1) / kRows; + parallel_for(row_tiles * column_tiles, threads, [&](size_t tile) { + const size_t row = (tile / column_tiles) * kRows; + const size_t column = (tile % column_tiles) * kColumns; + const size_t rows = std::min(kRows, m - row); + const size_t columns = std::min(kColumns, n - column); + std::array, kRows> accumulators{}; + for (size_t inner = 0; inner < k; ++inner) { + const float* weights = packed.data() + inner * n + column; + for (size_t i = 0; i < rows; ++i) { + const float activation = activations[(row + i) * k + inner]; + for (size_t j = 0; j < columns; ++j) { + accumulators[i][j] += activation * weights[j]; + } + } + } + for (size_t i = 0; i < rows; ++i) { + for (size_t j = 0; j < columns; ++j) { + output.values[(row + i) * n + column + j] = accumulators[i][j] + + (bias ? float(bias->values[column + j]) : 0.0f); + } + } + }); + return output; +} + +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/GemmReference.h b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/GemmReference.h new file mode 100644 index 00000000000..8b8caf57f47 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/GemmReference.h @@ -0,0 +1,18 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include + +namespace vkcompute::shader_profiling::reference { + +Tensor linear_projection( + const Tensor& input, + const Tensor& weight, + const Tensor* bias, + size_t threads); +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/GemmThroughputCommon.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/GemmThroughputCommon.cpp new file mode 100644 index 00000000000..b784b607d34 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/GemmThroughputCommon.cpp @@ -0,0 +1,44 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include + +namespace vkcompute::shader_profiling::throughput { + +double gemm_flops(const Json& spec, size_t weight_packing, size_t bias_index) { + if (weight_packing == 0) + throw std::invalid_argument("GEMM weight packing must be positive"); + const Json& input = throughput::tensor_argument(spec, 0); + const Json& weight = throughput::tensor_argument(spec, 1); + const Json& output = throughput::output_tensor(spec); + const auto input_sizes = input.at("sizes").get>(); + const auto weight_sizes = weight.at("sizes").get>(); + const auto output_sizes = output.at("sizes").get>(); + const double input_elements = throughput::tensor_elements(input); + const double output_elements = throughput::tensor_elements(output); + throughput::tensor_elements(weight); + if (weight_sizes.size() != 2 || input_sizes.empty() || output_sizes.empty()) { + throw std::invalid_argument( + "GEMM work count requires input/output ranks >= 1 and weight rank 2"); + } + const int64_t k = input_sizes.back(); + const int64_t n = output_sizes.back(); + if (weight_sizes[0] != n || weight_sizes[1] != k / weight_packing || + input_elements / k != output_elements / n || k % weight_packing != 0) { + throw std::invalid_argument("GEMM work count tensor dimensions disagree"); + } + const bool bias = !spec.at("args").at(bias_index).is_null(); + if (bias && + throughput::tensor_elements( + throughput::tensor_argument(spec, bias_index)) != n) { + throw std::invalid_argument("GEMM work count bias must have N elements"); + } + return output_elements * (2 * double(k) + (bias ? 1 : 0)); +} + +} // namespace vkcompute::shader_profiling::throughput diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/GemmThroughputCommon.h b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/GemmThroughputCommon.h new file mode 100644 index 00000000000..cb0a157ba82 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/GemmThroughputCommon.h @@ -0,0 +1,13 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once +#include + +namespace vkcompute::shader_profiling::throughput { + +double gemm_flops(const Json& spec, size_t weight_packing, size_t bias_index); +} // namespace vkcompute::shader_profiling::throughput diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/LinearReference.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/LinearReference.cpp new file mode 100644 index 00000000000..90a2f15f5c2 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/LinearReference.cpp @@ -0,0 +1,29 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include +#include + +namespace vkcompute::shader_profiling::reference { + +Tensor +linear_reference(const Json& spec, const TensorMap& tensors, size_t threads) { + const Tensor* bias = + spec.at("args").at(2).is_null() ? nullptr : &tensor_arg(spec, 2, tensors); + return linear_projection( + tensor_arg(spec, 0, tensors), + tensor_arg(spec, 1, tensors), + bias, + threads); +} + +REGISTER_REFERENCE_FUNCTIONS { + REGISTER_REFERENCE_FN(linear, linear_reference); + REGISTER_OPERATOR_REFERENCE(aten.linear.default, linear); +} + +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/LinearReference.h b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/LinearReference.h new file mode 100644 index 00000000000..958eed7ccea --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/LinearReference.h @@ -0,0 +1,15 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include + +namespace vkcompute::shader_profiling::reference { + +Tensor +linear_reference(const Json& spec, const TensorMap& tensors, size_t threads); +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/LinearThroughput.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/LinearThroughput.cpp new file mode 100644 index 00000000000..d030700c0e6 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/LinearThroughput.cpp @@ -0,0 +1,62 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include + +namespace vkcompute::shader_profiling { +namespace { + +double linear_flops(const Json& spec) { + return throughput::gemm_flops(spec, 1, 2); +} + +double linear_shader_flops(const Json& spec) { + const Json* input = nullptr; + for (const auto& binding : spec.at("bindings").items()) { + const Json& tensor = binding.value(); + if (tensor.contains("sizes") && tensor.value("access", "") == "read" && + (!input || + tensor.value("binding_index", SIZE_MAX) < + input->value("binding_index", SIZE_MAX))) { + input = &tensor; + } + } + if (!input) { + throw std::invalid_argument( + "Linear shader work count requires first input binding"); + } + const Json& output = throughput::output_tensor(spec); + const double output_elements = throughput::tensor_elements(output); + const double input_elements = throughput::tensor_elements(*input); + const auto input_sizes = input->at("sizes").get>(); + const auto output_sizes = output.at("sizes").get>(); + if (input_elements / input_sizes.back() != + output_elements / output_sizes.back()) { + throw std::invalid_argument( + "Linear shader input/output row counts disagree"); + } + return output_elements * 2 * double(input_sizes.back()); +} + +double linear_bias_shader_flops(const Json& spec) { + return linear_shader_flops(spec) + + throughput::tensor_elements(throughput::output_tensor(spec)); +} + +} // namespace + +REGISTER_THROUGHPUT_FUNCTIONS { + REGISTER_OPERATOR_MEMORY_TRAFFIC( + aten.linear.default, throughput::tensor_traffic); + REGISTER_OPERATOR_COMPUTE_FLOPS(aten.linear.default, linear_flops); + REGISTER_SHADER_MEMORY_TRAFFIC(linear_vec, throughput::tensor_traffic); + REGISTER_SHADER_COMPUTE_FLOPS(linear_vec, linear_shader_flops); + REGISTER_SHADER_COMPUTE_FLOPS(linear_vec_bias, linear_bias_shader_flops); +} + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/q8ta/Q8taLinearReference.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/q8ta/Q8taLinearReference.cpp new file mode 100644 index 00000000000..1573db58e0d --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/q8ta/Q8taLinearReference.cpp @@ -0,0 +1,75 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include +#include +namespace vkcompute::shader_profiling::reference { +static Tensor q8ta_linear_reference( + const Json& spec, + const TensorMap& tensors, + size_t threads) { + const Tensor& input = tensor_arg(spec, 0, tensors); + const Tensor& weight = tensor_arg(spec, 3, tensors); + const Tensor& weight_sums = tensor_arg(spec, 4, tensors); + const Tensor& scales = tensor_arg(spec, 5, tensors); + const Tensor* bias = + spec.at("args").at(8).is_null() ? nullptr : &tensor_arg(spec, 8, tensors); + require( + !input.sizes.empty() && weight.sizes.size() == 2, + "Q8 linear requires input rank >= 1 and matrix weights"); + const int64_t k = input.sizes.back(), n = weight.sizes[0]; + require(k > 0 && n > 0, "Q8 linear requires positive feature dimensions"); + const int64_t m = int64_t(input.values.size()) / k; + require(weight.sizes[1] == k, "Q8 linear weight/input features disagree"); + auto out_sizes = input.sizes; + out_sizes.back() = n; + Tensor out{std::move(out_sizes), std::vector(size_t(m * n))}; + const float is = q8ta::scale(spec, 1), os = q8ta::scale(spec, 6); + const int64_t iz = q8ta::integer(spec, 2), oz = q8ta::integer(spec, 7); + std::vector quantized(input.values.size()); + for (size_t i = 0; i < quantized.size(); ++i) + quantized[i] = int16_t(input.values[i]); + std::vector packed_weight(weight.values.size()); + for (int64_t col = 0; col < n; ++col) + for (int64_t j = 0; j < k; ++j) + packed_weight[size_t(j * n + col)] = + int8_t(weight.values[size_t(col * k + j)]); + constexpr int64_t tile = 32; + const int64_t blocks = (n + tile - 1) / tile; + parallel_for(size_t(m * blocks), threads, [&](size_t index) { + const int64_t row = int64_t(index) / blocks, + start = int64_t(index) % blocks * tile, + count = std::min(tile, n - start); + std::array sums{}; + for (int64_t j = 0; j < k; ++j) { + const int64_t value = quantized[size_t(row * k + j)]; + for (int64_t col = 0; col < count; ++col) + sums[size_t(col)] += value * packed_weight[size_t(j * n + start + col)]; + } + for (int64_t col = 0; col < count; ++col) { + const size_t oc = size_t(start + col); + const int64_t corrected = + sums[size_t(col)] - iz * int64_t(weight_sums.values[oc]); + out.values[size_t(row * n) + oc] = double(q8ta::requantize_accumulator( + corrected, + is, + float(scales.values[oc]), + os, + oz, + bias ? float(bias->values[oc]) : 0.0f, + spec.at("args").at(9).at("string") == "relu", + false)); + } + }); + return out; +} +REGISTER_REFERENCE_FUNCTIONS { + REGISTER_REFERENCE_FN(q8ta_linear, q8ta_linear_reference); + REGISTER_OPERATOR_REFERENCE(et_vk.q8ta_linear.default, q8ta_linear); + REGISTER_OPERATOR_REFERENCE(et_vk.q8ta_linear_gemv.default, q8ta_linear); +} +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/q8ta/Q8taLinearRoutes.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/q8ta/Q8taLinearRoutes.cpp new file mode 100644 index 00000000000..1c61a982115 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/q8ta/Q8taLinearRoutes.cpp @@ -0,0 +1,24 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include + +namespace vkcompute::shader_profiling { +static Json q8ta_linear_gemv_route( + ComputeGraph& graph, + const std::vector& args, + const Json& options) { + if (!options.empty()) + throw std::invalid_argument("Q8 linear gemv route accepts no options"); + VK_GET_OP_FN("et_vk.q8ta_linear_gemv.default")(graph, args); + return {{"name", "gemv"}, {"options", Json::object()}}; +} +REGISTER_OPERATOR_ROUTES { + REGISTER_OPERATOR_ROUTE( + et_vk.q8ta_linear.default, gemv, q8ta_linear_gemv_route); +} +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/q8ta/Q8taLinearThroughput.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/q8ta/Q8taLinearThroughput.cpp new file mode 100644 index 00000000000..b6c28cc2a79 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/q8ta/Q8taLinearThroughput.cpp @@ -0,0 +1,26 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +namespace vkcompute::shader_profiling { +namespace { +double q8ta_linear_flops(const Json& spec) { + const auto& input = throughput::tensor_argument(spec, 0); + const double k = input.at("sizes").back().get(); + return 2 * k * throughput::tensor_elements(throughput::output_tensor(spec)); +} +REGISTER_THROUGHPUT_FUNCTIONS { + REGISTER_OPERATOR_MEMORY_TRAFFIC( + et_vk.q8ta_linear.default, throughput::tensor_traffic); + REGISTER_OPERATOR_MEMORY_TRAFFIC( + et_vk.q8ta_linear_gemv.default, throughput::tensor_traffic); + REGISTER_OPERATOR_COMPUTE_FLOPS(et_vk.q8ta_linear.default, q8ta_linear_flops); + REGISTER_OPERATOR_COMPUTE_FLOPS( + et_vk.q8ta_linear_gemv.default, q8ta_linear_flops); +} +} // namespace +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/q8ta/cases/correctness.json b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/q8ta/cases/correctness.json new file mode 100644 index 00000000000..f9c5f4ebde9 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/q8ta/cases/correctness.json @@ -0,0 +1,1638 @@ +[ + { + "name": "q8ta/linear/correctness [0]", + "operator": "et_vk.q8ta_linear.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 64], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/linear/correctness [1]", + "operator": "et_vk.q8ta_linear.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "gemv", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 64], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/linear/correctness [2]", + "operator": "et_vk.q8ta_linear.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 128], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/linear/correctness [3]", + "operator": "et_vk.q8ta_linear.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "gemv", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 128], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/linear/correctness [4]", + "operator": "et_vk.q8ta_linear.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 256], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/linear/correctness [5]", + "operator": "et_vk.q8ta_linear.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "gemv", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 256], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/linear/correctness [6]", + "operator": "et_vk.q8ta_linear.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 128], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + null, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/linear/correctness [7]", + "operator": "et_vk.q8ta_linear.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "gemv", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 128], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + null, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/linear/correctness [8]", + "operator": "et_vk.q8ta_linear.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 64], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [4, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/linear/correctness [9]", + "operator": "et_vk.q8ta_linear.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 128], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [4, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/linear/correctness [10]", + "operator": "et_vk.q8ta_linear.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4, 256], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [4, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/linear/correctness [11]", + "operator": "et_vk.q8ta_linear.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 64], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [32, 32], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/linear/correctness [12]", + "operator": "et_vk.q8ta_linear.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 128], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [32, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/linear/correctness [13]", + "operator": "et_vk.q8ta_linear.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 256], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [32, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/linear/correctness [14]", + "operator": "et_vk.q8ta_linear.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 128], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + null, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [32, 64], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/linear/correctness [15]", + "operator": "et_vk.q8ta_linear.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 256], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + null, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [32, 128], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/q8ta/cases/performance.json b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/q8ta/cases/performance.json new file mode 100644 index 00000000000..6379d8c2073 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/q8ta/cases/performance.json @@ -0,0 +1,947 @@ +[ + { + "name": "q8ta/linear/performance [0]", + "operator": "et_vk.q8ta_linear.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 512], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [512, 512], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [512], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [512], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [512], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 512], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/linear/performance [1]", + "operator": "et_vk.q8ta_linear.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "gemv", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 512], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [512, 512], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [512], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [512], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [512], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 512], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/linear/performance [2]", + "operator": "et_vk.q8ta_linear.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 2048], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [2048, 2048], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [2048], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [2048], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [2048], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 2048], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/linear/performance [3]", + "operator": "et_vk.q8ta_linear.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "gemv", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 2048], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [2048, 2048], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [2048], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [2048], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [2048], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 2048], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/linear/performance [4]", + "operator": "et_vk.q8ta_linear.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 512], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [9059, 512], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [9059], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [9059], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [9059], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 9059], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/linear/performance [5]", + "operator": "et_vk.q8ta_linear.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "gemv", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 512], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [9059, 512], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [9059], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [9059], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [9059], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1, 9059], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/linear/performance [6]", + "operator": "et_vk.q8ta_linear.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [256, 2048], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [2048, 2048], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [2048], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [2048], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [2048], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [256, 2048], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/linear/performance [7]", + "operator": "et_vk.q8ta_linear.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [512, 2048], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [2048, 2048], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [2048], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [2048], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [2048], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [512, 2048], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/linear/performance [8]", + "operator": "et_vk.q8ta_linear.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0018818534756524, + "rtol": 0 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1024, 2048], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [2048, 2048], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [2048], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [2048], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "float": 0.05313999950885773 + }, + { + "int": -1 + }, + { + "tensor": { + "name": "bias", + "sizes": [2048], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "string": "none" + }, + { + "tensor": { + "name": "output", + "sizes": [1024, 2048], + "dtype": "int8x4", + "storage": "buffer", + "layout": "4h4w", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/dq8ca_q4gsw/Dq8caQ4gswReference.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/dq8ca_q4gsw/Dq8caQ4gswReference.cpp new file mode 100644 index 00000000000..168db9c689c --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/dq8ca_q4gsw/Dq8caQ4gswReference.cpp @@ -0,0 +1,59 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include +#include +#include +#include +#include +namespace vkcompute::shader_profiling::reference { +Tensor dq8ca_q4gsw_reference( + const Json& spec, + const TensorMap& tensors, + size_t threads) { + Json canonical = spec; + canonical["args"] = { + spec.at("args").at(0), + spec.at("args").at(3), + spec.at("args").at(5), + spec.at("args").at(6), + spec.at("args").at(7), + spec.at("args").at(8)}; + const Json context = spec.value("reference_context", Json::object()); + require(context.is_object(), "Reference context must be an object"); + const Json dot = context.value("int8_dot_product", Json(true)); + require(dot.is_boolean(), "DQ8CA reference capability must be boolean"); + if (!dot.get()) { + return q4gsw_linear_reference(canonical, tensors, threads); + } + TensorMap converted = tensors; + const std::string name = spec.at("args").at(0).at("tensor"); + Tensor& input = converted.at(name); + const Tensor& scales = tensor_arg(spec, 1, tensors); + const Tensor& zeros = tensor_arg(spec, 2, tensors); + const size_t k = input.sizes.back(), m = input.values.size() / k; + require( + scales.values.size() == m && zeros.values.size() == m, + "Dynamic Q4 activation qparams must be per row"); + parallel_for(m, threads, [&](size_t row) { + const float scale = scales.values[row], zp = zeros.values[row]; + require(scale > 0, "Activation scale must be positive"); + for (size_t i = 0; i < k; ++i) { + const float q = std::clamp( + std::round(float(input.values[row * k + i]) / scale) + zp, + -128.0f, + 127.0f); + input.values[row * k + i] = (q - zp) * scale; + } + }); + return q4gsw_linear_reference(canonical, converted, threads); +} +REGISTER_REFERENCE_FUNCTIONS { + REGISTER_REFERENCE_FN(dq8ca_q4gsw, dq8ca_q4gsw_reference); + REGISTER_OPERATOR_REFERENCE(et_vk.linear_dq8ca_q4gsw.default, dq8ca_q4gsw); +} +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/dq8ca_q4gsw/Dq8caQ4gswReference.h b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/dq8ca_q4gsw/Dq8caQ4gswReference.h new file mode 100644 index 00000000000..2aeb21eb299 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/dq8ca_q4gsw/Dq8caQ4gswReference.h @@ -0,0 +1,18 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include + +namespace vkcompute::shader_profiling::reference { + +Tensor dq8ca_q4gsw_reference( + const Json& spec, + const TensorMap& tensors, + size_t threads); + +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/dq8ca_q4gsw/Dq8caQ4gswThroughput.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/dq8ca_q4gsw/Dq8caQ4gswThroughput.cpp new file mode 100644 index 00000000000..27fc7ab8b4d --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/dq8ca_q4gsw/Dq8caQ4gswThroughput.cpp @@ -0,0 +1,26 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +namespace vkcompute::shader_profiling { +namespace { +double quantized_linear_legacy_flops(const Json& spec) { + const auto sizes = throughput::tensor_argument(spec, 0) + .at("sizes") + .get>(); + return throughput::tensor_elements(throughput::output_tensor(spec)) * + (3.0 * sizes.back() + 1); +} +REGISTER_THROUGHPUT_FUNCTIONS { + REGISTER_OPERATOR_MEMORY_TRAFFIC( + et_vk.linear_dq8ca_q4gsw.default, throughput::tensor_traffic); + REGISTER_OPERATOR_COMPUTE_FLOPS( + et_vk.linear_dq8ca_q4gsw.default, quantized_linear_legacy_flops); +} + +} // namespace +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/Q4gswLinearReference.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/Q4gswLinearReference.cpp new file mode 100644 index 00000000000..12394c6f58d --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/Q4gswLinearReference.cpp @@ -0,0 +1,56 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include +#include + +namespace vkcompute::shader_profiling::reference { + +Tensor q4gsw_linear_reference( + const Json& spec, + const TensorMap& tensors, + size_t threads) { + const Tensor& input = tensor_arg(spec, 0, tensors); + const Tensor& packed = tensor_arg(spec, 1, tensors); + const Tensor& scales = tensor_arg(spec, 2, tensors); + require( + !input.sizes.empty() && packed.sizes.size() == 2, + "Q4 linear requires input rank >= 1 and packed weight rank 2"); + const size_t k = input.sizes.back(); + const size_t n = packed.sizes[0]; + const int64_t group_size = spec.at("args").at(3).at("int"); + require( + group_size > 0 && k % group_size == 0 && k % 2 == 0, + "Q4 linear requires even K divisible by group_size"); + require( + size_t(packed.sizes[1]) == k / 2, + "Q4 packed weight shape must be [N, K/2]"); + require( + scales.sizes == + std::vector({int64_t(k / group_size), int64_t(n)}), + "Q4 scale shape must be [K/group_size, N]"); + Tensor weight{{int64_t(n), int64_t(k)}, std::vector(n * k)}; + parallel_for(n, threads, [&](size_t column) { + for (size_t inner = 0; inner < k; ++inner) { + const uint8_t byte = packed.values[column * (k / 2) + inner / 2]; + const int value = ((byte >> (4 * (inner % 2))) & 15) - 8; + weight.values[column * k + inner] = float(value) * + float(scales.values[(inner / group_size) * n + column]); + } + }); + const Tensor* bias = + spec.at("args").at(4).is_null() ? nullptr : &tensor_arg(spec, 4, tensors); + return linear_projection(input, weight, bias, threads); +} + +REGISTER_REFERENCE_FUNCTIONS { + REGISTER_REFERENCE_FN(q4gsw_linear, q4gsw_linear_reference); + REGISTER_OPERATOR_REFERENCE(et_vk.q4gsw_linear.default, q4gsw_linear); + REGISTER_OPERATOR_REFERENCE(et_vk.linear_q4gsw.default, q4gsw_linear); +} + +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/Q4gswLinearReference.h b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/Q4gswLinearReference.h new file mode 100644 index 00000000000..75b4846d686 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/Q4gswLinearReference.h @@ -0,0 +1,17 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include + +namespace vkcompute::shader_profiling::reference { + +Tensor q4gsw_linear_reference( + const Json& spec, + const TensorMap& tensors, + size_t threads); +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/Q4gswLinearThroughput.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/Q4gswLinearThroughput.cpp new file mode 100644 index 00000000000..66b388943f5 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/Q4gswLinearThroughput.cpp @@ -0,0 +1,91 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include + +namespace vkcompute::shader_profiling { +namespace { + +double q4gsw_linear_flops(const Json& spec) { + const double flops = throughput::gemm_flops(spec, 2, 4); + const auto input_sizes = throughput::tensor_argument(spec, 0) + .at("sizes") + .get>(); + const auto output_sizes = + throughput::output_tensor(spec).at("sizes").get>(); + const int64_t k = input_sizes.back(); + const int64_t n = output_sizes.back(); + + const int64_t group_size = spec.at("args").at(3).at("int"); + const Json& scales = throughput::tensor_argument(spec, 2); + const auto scale_sizes = scales.at("sizes").get>(); + if (group_size <= 0 || k % group_size != 0 || + scale_sizes != std::vector({k / group_size, n})) { + throw std::invalid_argument( + "Q4 work count requires scales [K/group_size, N]"); + } + return flops; +} + +double q4_shader_traffic(const Json& spec, size_t inactive_input) { + Json filtered = spec; + Json& bindings = filtered.at("bindings"); + unsigned seen_indices = 0; + for (auto binding = bindings.begin(); binding != bindings.end();) { + const Json& tensor = binding.value(); + if (!tensor.contains("sizes")) { + ++binding; + continue; + } + const size_t index = tensor.at("binding_index").get(); + if (index >= 6 || (seen_indices & (1u << index))) { + throw std::invalid_argument( + "Q4 shader memory count requires unique tensor binding indices 0..5"); + } + seen_indices |= 1u << index; + if (index == inactive_input || (index == 5 && tensor.at("sizes").empty())) { + binding = bindings.erase(binding); + } else { + ++binding; + } + } + if (seen_indices != 63) { + throw std::invalid_argument( + "Q4 shader memory count requires all six tensor binding slots"); + } + return throughput::tensor_traffic(filtered); +} + +double q4_gemm_shader_traffic(const Json& spec) { + return q4_shader_traffic(spec, 2); +} + +double q4_tin_shader_traffic(const Json& spec) { + return q4_shader_traffic(spec, 1); +} + +} // namespace + +REGISTER_THROUGHPUT_FUNCTIONS { + REGISTER_OPERATOR_MEMORY_TRAFFIC( + et_vk.q4gsw_linear.default, throughput::tensor_traffic); + REGISTER_OPERATOR_COMPUTE_FLOPS( + et_vk.q4gsw_linear.default, q4gsw_linear_flops); + REGISTER_OPERATOR_MEMORY_TRAFFIC( + et_vk.linear_q4gsw.default, throughput::tensor_traffic); + REGISTER_OPERATOR_COMPUTE_FLOPS( + et_vk.linear_q4gsw.default, q4gsw_linear_flops); + REGISTER_SHADER_MEMORY( + transpose_cast_contig_to_vectorized_half_buffer_half_buffer, + throughput::tensor_traffic); + REGISTER_SHADER_MEMORY_TRAFFIC(q4gsw_linear_gemm__, q4_gemm_shader_traffic); + REGISTER_SHADER_MEMORY_TRAFFIC( + q4gsw_linear_gemm__tin__, q4_tin_shader_traffic); +} + +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/Q4gswRoutes.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/Q4gswRoutes.cpp new file mode 100644 index 00000000000..47df7a3d394 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/Q4gswRoutes.cpp @@ -0,0 +1,766 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace vkcompute { + +namespace { + +enum class TestKernelKind { + PROD, + GEMM_W_4X8, + GEMM_TIN_W_4X8, + GEMV_W_4X8, + GEMV_W_4X8_NOSG, + LEGACY, + GEMV_COOP_W_4X8_NC_BUFFER, + + GEMV_COOP_W_4X8_NC_BUFFER_G1W64, + GEMV_COOP_W_4X8_NC_BUFFER_G4W16, + GEMV_COOP_W_4X8_NC_BUFFER_G8W8, +}; + +TestKernelKind selector_to_kind(int32_t selector, bool is_gemv) { + if (is_gemv) { + switch (selector) { + case 0: + return TestKernelKind::PROD; + case 1: + return TestKernelKind::GEMV_W_4X8; + case 2: + return TestKernelKind::GEMV_W_4X8_NOSG; + case 3: + return TestKernelKind::LEGACY; + case 13: + return TestKernelKind::GEMV_COOP_W_4X8_NC_BUFFER; + case 14: + return TestKernelKind::GEMV_COOP_W_4X8_NC_BUFFER_G1W64; + case 15: + return TestKernelKind::GEMV_COOP_W_4X8_NC_BUFFER_G4W16; + case 16: + return TestKernelKind::GEMV_COOP_W_4X8_NC_BUFFER_G8W8; + default: + return TestKernelKind::PROD; + } + } + switch (selector) { + case 0: + return TestKernelKind::PROD; + case 1: + return TestKernelKind::GEMM_W_4X8; + case 2: + return TestKernelKind::GEMM_TIN_W_4X8; + case 3: + return TestKernelKind::LEGACY; + default: + return TestKernelKind::PROD; + } +} + +const char* forced_kind_base_name(TestKernelKind kind) { + switch (kind) { + case TestKernelKind::GEMM_W_4X8: + return "q4gsw_linear_gemm__w_4x8_nc"; + case TestKernelKind::GEMM_TIN_W_4X8: + return "q4gsw_linear_gemm__tin__w_4x8_nc"; + case TestKernelKind::GEMV_W_4X8: + return "q4gsw_linear_gemv__w_4x8_nc"; + case TestKernelKind::GEMV_W_4X8_NOSG: + return "q4gsw_linear_gemv__w_4x8_nc_nosg"; + case TestKernelKind::GEMV_COOP_W_4X8_NC_BUFFER: + case TestKernelKind::GEMV_COOP_W_4X8_NC_BUFFER_G1W64: + return "q4gsw_linear_gemv_coop__w_4x8_nc_buffer_g1w64"; + case TestKernelKind::GEMV_COOP_W_4X8_NC_BUFFER_G4W16: + return "q4gsw_linear_gemv_coop__w_4x8_nc_buffer_g4w16"; + case TestKernelKind::GEMV_COOP_W_4X8_NC_BUFFER_G8W8: + return "q4gsw_linear_gemv_coop__w_4x8_nc_buffer_g8w8"; + case TestKernelKind::PROD: + case TestKernelKind::LEGACY: + default: + return ""; + } +} + +template +vkapi::ShaderInfo pick_forced_shader( + ComputeGraph* graph, + const std::vector& args, + const std::vector& resize_args) { + (void)resize_args; + const ValueRef out = args.at(0).refs.at(0); + std::string kernel_name = forced_kind_base_name(KIND); + add_storage_type_suffix(kernel_name, graph->storage_type_of(out)); + add_dtype_suffix(kernel_name, graph->dtype_of(out)); + return VK_KERNEL_FROM_STR(kernel_name); +} + +template +vkapi::ShaderInfo pick_forced_shader_coop_kc( + ComputeGraph* graph, + const std::vector& args, + const std::vector& resize_args) { + (void)resize_args; + const ValueRef out = args.at(0).refs.at(0); + std::string kernel_name = forced_kind_base_name(KIND); + add_storage_type_suffix(kernel_name, graph->storage_type_of(out)); + add_dtype_suffix(kernel_name, graph->dtype_of(out)); + return VK_KERNEL_FROM_STR(kernel_name); +} + +template +GlobalWorkGrid pick_q4gsw_coop_gwg( + ComputeGraph* graph, + const vkapi::ShaderInfo& shader, + const std::vector& args, + const std::vector& resize_args) { + (void)shader; + (void)resize_args; + const ValueRef out = args.at(0).refs.at(0); + const std::vector out_sizes = graph->sizes_of(out); + const uint32_t N = + utils::safe_downcast(utils::val_at(-1, out_sizes)); + const uint32_t N8 = (N + 7u) / 8u; + const uint32_t wgs_along_x = utils::div_up(N8, NUM_GROUPS); + const LocalWorkGroup lwg(1u, NUM_GROUPS, WORKERS_PER_GROUP); + GlobalWorkGrid gwg({wgs_along_x * NUM_GROUPS, 1u, 1u}, kLinearWorkGrid); + gwg.wrap_linear_dispatch( + graph->context()->adapter_ptr()->max_compute_workgroup_count(), lwg); + return gwg; +} + +template +LocalWorkGroup pick_q4gsw_coop_lwg( + ComputeGraph* graph, + const vkapi::ShaderInfo& shader, + const GlobalWorkGrid& gwg, + const std::vector& args, + const std::vector& resize_args) { + (void)graph; + (void)shader; + (void)gwg; + (void)args; + (void)resize_args; + return LocalWorkGroup(1u, NUM_GROUPS, WORKERS_PER_GROUP); +} + +constexpr uint32_t kGemvSubgroupSize = 64u; +constexpr uint32_t kGemvNumSubgroups = 4u; + +GlobalWorkGrid pick_q4gsw_legacy_gemv_gwg( + ComputeGraph* graph, + const vkapi::ShaderInfo& shader, + const std::vector& args, + const std::vector& resize_args) { + (void)shader; + (void)resize_args; + const ValueRef out = args.at(0).refs.at(0); + const uint32_t N = + utils::safe_downcast(utils::val_at(-1, graph->sizes_of(out))); + + return GlobalWorkGrid({N / 2u, kGemvNumSubgroups, 1u}, kTiledWorkGrid); +} + +LocalWorkGroup pick_q4gsw_legacy_gemv_lwg( + ComputeGraph* graph, + const vkapi::ShaderInfo& shader, + const GlobalWorkGrid& gwg, + const std::vector& args, + const std::vector& resize_args) { + (void)graph; + (void)shader; + (void)gwg; + (void)args; + (void)resize_args; + return LocalWorkGroup(kGemvSubgroupSize, kGemvNumSubgroups, 1u); +} + +void legacy_q4gsw_resize_linear_node( + ComputeGraph* graph, + const std::vector& args, + const std::vector& extra_args) { + (void)extra_args; + + ValueRef output = args.at(0).refs.at(0); + ValueRef fp_input = args.at(1).refs.at(0); + ValueRef weight_data = extra_args.at(1); + + std::vector mat1_sizes = graph->sizes_of(fp_input); + std::vector mat2_sizes = graph->sizes_of(weight_data); + + const int64_t out_cols = utils::val_at(-2, mat1_sizes); + const int64_t out_rows = utils::val_at(-2, mat2_sizes); + + std::vector new_out_sizes(3); + if (mat1_sizes.size() == 2) { + new_out_sizes.resize(2); + new_out_sizes.at(0) = out_cols; + new_out_sizes.at(1) = out_rows; + } else { + new_out_sizes.at(0) = mat1_sizes.at(0); + new_out_sizes.at(1) = out_cols; + new_out_sizes.at(2) = out_rows; + } + + graph->virtual_resize(output, new_out_sizes); +} + +GlobalWorkGrid legacy_q4gsw_gwg( + ComputeGraph* graph, + const vkapi::ShaderInfo& shader, + const std::vector& args, + const std::vector& resize_args) { + (void)resize_args; + const ValueRef out = args.at(0).refs.at(0); + + std::vector out_sizes = graph->sizes_of(out); + + const uint32_t N = + utils::safe_downcast(utils::val_at(-1, out_sizes)); + + const uint32_t M = + utils::safe_downcast(utils::val_at(-2, out_sizes)); + + uint32_t N_per_tile = 8; + uint32_t M_per_tile = 4; + if (shader.kernel_name.find("coop") != std::string::npos) { + M_per_tile = 1; + } + + const uint32_t num_N_tiles = utils::div_up(N, N_per_tile); + const uint32_t num_M_tiles = utils::div_up(M, M_per_tile); + + return GlobalWorkGrid({num_N_tiles, num_M_tiles, 1u}, kTiledWorkGrid); +} + +LocalWorkGroup legacy_q4gsw_lwg( + ComputeGraph* graph, + const vkapi::ShaderInfo& shader, + const GlobalWorkGrid& gwg, + const std::vector& args, + const std::vector& resize_args) { + const bool use_coop_algorithm = + shader.kernel_name.find("_coop") != std::string::npos; + + if (use_coop_algorithm) { + return LocalWorkGroup(1u, 1u, 64u); + } + return pick_xy_square_lwg(graph, shader, gwg, args, resize_args); +} + +vkapi::ShaderInfo legacy_q4gsw_pick_shader( + ComputeGraph* graph, + const std::vector& args, + const std::vector& resize_args) { + (void)resize_args; + const ValueRef output = args.at(0).refs.at(0); + const ValueRef fp_input = args.at(1).refs.at(0); + const ValueRef packed_int_weight = args.at(1).refs.at(1); + + const bool is_gemv_case = is_gemv(graph, fp_input); + + std::string kernel_name = "linear_q4gsw"; + kernel_name += is_gemv_case ? "_coop" : "_tiled"; + + add_storage_type_suffix(kernel_name, graph->storage_type_of(output)); + add_storage_type_suffix( + kernel_name, graph->storage_type_of(packed_int_weight)); + add_dtype_suffix(kernel_name, graph->dtype_of(output)); + + return VK_KERNEL_FROM_STR(kernel_name); +} + +ValueRef legacy_prepack_q4gsw_weight( + ComputeGraph& graph, + const ValueRef qmat2_data) { + std::vector qmat2_orig_sizes = graph.sizes_of(qmat2_data); + const int64_t ndim = graph.dim_of(qmat2_data); + + const int64_t qmat2_width = qmat2_orig_sizes.at(ndim - 1); + const int64_t qmat2_height = qmat2_orig_sizes.at(ndim - 2); + + const int64_t K = qmat2_width * 2; + const int64_t N = qmat2_height; + + VK_CHECK_COND(K % 8 == 0); + + const int64_t N_per_block = 8; + const int64_t K_per_block = 4; + + const int64_t num_blocks_K = utils::div_up(K, K_per_block); + const int64_t num_blocks_N = utils::div_up(N, N_per_block); + + const int64_t output_height = num_blocks_N; + const int64_t output_width = num_blocks_K * 4; + + utils::ivec2 orig_sizes = { + utils::safe_downcast(K), utils::safe_downcast(N)}; + + std::vector qmat2_sizes{output_height, output_width}; + + utils::StorageType storage_type = utils::kTexture2D; + uint32_t max_extent = graph.context()->adapter_ptr()->max_texture2d_dim(); + if (output_width > max_extent * 4 || output_height > max_extent) { + storage_type = utils::kBuffer; + } + + std::string kernel_name = "pack_q4_linear_weight"; + add_storage_type_suffix(kernel_name, storage_type); + + ValueRef cached = graph.get_cached_prepack(qmat2_data, kernel_name); + if (is_valid(cached)) { + return cached; + } + + ValueRef qmat2 = graph.add_tensor( + qmat2_sizes, vkapi::kInt, storage_type, utils::kWidthPacked); + + const GlobalWorkGrid gwg( + {utils::safe_downcast(utils::div_up(num_blocks_K, int64_t(2))), + utils::safe_downcast(num_blocks_N), + 1u}, + kTiledWorkGrid); + + graph.prepack_nodes().emplace_back(new PrepackNode( + graph, + VK_KERNEL_FROM_STR(kernel_name), + gwg, + graph.create_lwg(gwg), + qmat2_data, + qmat2, + + {}, + + {}, + + {graph.sizes_pc_of(qmat2), + PushConstantDataInfo(&orig_sizes, sizeof(utils::ivec2))})); + + graph.cache_prepack(qmat2_data, kernel_name, qmat2); + return qmat2; +} + +void add_legacy_q4gsw_linear_node( + ComputeGraph& graph, + const ValueRef fp_input, + const ValueRef weight_data, + const ValueRef weight_scales_data, + const ValueRef group_size_ref, + const ValueRef bias_data, + const ValueRef output) { + std::vector input_sizes = graph.sizes_of(fp_input); + const int64_t K = utils::val_at(-1, input_sizes); + + VK_CHECK_COND(K % 4 == 0); + + const ValueRef packed_weight = + legacy_prepack_q4gsw_weight(graph, weight_data); + const ValueRef packed_weight_scales = prepack_standard( + graph, weight_scales_data, utils::kBuffer, utils::kWidthPacked); + + TmpTensor dummy_bias( + &graph, {}, graph.dtype_of(output), utils::kBuffer, utils::kWidthPacked); + ValueRef packed_bias = dummy_bias.vref; + uint32_t apply_bias = 0; + if (graph.val_is_not_none(bias_data)) { + packed_bias = + prepack_standard(graph, bias_data, utils::kBuffer, utils::kWidthPacked); + apply_bias = 1; + } + + const int32_t group_size_val = graph.extract_scalar(group_size_ref); + const int32_t K4_per_group = utils::div_up(group_size_val, int32_t(4)); + + vkapi::ParamsBindList param_buffers = { + graph.sizes_ubo(output), graph.sizes_ubo(fp_input)}; + + graph.execute_nodes().emplace_back(new DynamicDispatchNode( + graph, + legacy_q4gsw_pick_shader, + legacy_q4gsw_gwg, + legacy_q4gsw_lwg, + + {{output, vkapi::kWrite}, + {{fp_input, packed_weight, packed_weight_scales, packed_bias}, + vkapi::kRead}}, + + param_buffers, + + {}, + + {apply_bias, K4_per_group}, + + {kDummyValueRef, weight_data}, + legacy_q4gsw_resize_linear_node)); +} + +void add_q4gsw_linear_coop_kc_forced_node( + ComputeGraph& graph, + const ValueRef fp_input, + const ValueRef weight_data, + const ValueRef weight_scales_data, + const ValueRef group_size_ref, + const ValueRef bias_data, + const ValueRef output, + TestKernelKind kind) { + const vkapi::ScalarType in_dtype = graph.dtype_of(fp_input); + + const int64_t group_size_val = graph.extract_scalar(group_size_ref); + + std::vector weight_sizes = graph.sizes_of(weight_data); + const int64_t K = weight_sizes.at(1) * 2; + const uint32_t K_val = static_cast(K); + + const ValueRef packed_weight_kc = + prepack_q4_w_4x8_nc_buffer(graph, weight_data); + const ValueRef packed_scales = + prepack_q4_scales(graph, weight_scales_data, in_dtype); + + TmpTensor dummy_bias( + &graph, {}, graph.dtype_of(output), utils::kBuffer, utils::kWidthPacked); + ValueRef packed_bias = dummy_bias.vref; + uint32_t apply_bias = 0; + if (graph.val_is_not_none(bias_data)) { + packed_bias = + prepack_standard(graph, bias_data, utils::kBuffer, utils::kWidthPacked); + apply_bias = 1; + } + + TmpTensor dummy_transposed_input( + &graph, {}, in_dtype, utils::kBuffer, utils::kWidthPacked); + + using PickShaderFn = vkapi::ShaderInfo (*)( + ComputeGraph*, + const std::vector&, + const std::vector&); + using PickGwgFn = GlobalWorkGrid (*)( + ComputeGraph*, + const vkapi::ShaderInfo&, + const std::vector&, + const std::vector&); + using PickLwgFn = LocalWorkGroup (*)( + ComputeGraph*, + const vkapi::ShaderInfo&, + const GlobalWorkGrid&, + const std::vector&, + const std::vector&); + + PickShaderFn pick_shader = nullptr; + PickGwgFn pick_gwg = nullptr; + PickLwgFn pick_lwg = nullptr; + + switch (kind) { + case TestKernelKind::GEMV_COOP_W_4X8_NC_BUFFER: + case TestKernelKind::GEMV_COOP_W_4X8_NC_BUFFER_G1W64: + pick_shader = + pick_forced_shader_coop_kc; + pick_gwg = pick_q4gsw_coop_gwg<1u, 64u>; + pick_lwg = pick_q4gsw_coop_lwg<1u, 64u>; + break; + case TestKernelKind::GEMV_COOP_W_4X8_NC_BUFFER_G4W16: + pick_shader = pick_forced_shader_coop_kc< + TestKernelKind::GEMV_COOP_W_4X8_NC_BUFFER_G4W16>; + pick_gwg = pick_q4gsw_coop_gwg<4u, 16u>; + pick_lwg = pick_q4gsw_coop_lwg<4u, 16u>; + break; + case TestKernelKind::GEMV_COOP_W_4X8_NC_BUFFER_G8W8: + pick_shader = pick_forced_shader_coop_kc< + TestKernelKind::GEMV_COOP_W_4X8_NC_BUFFER_G8W8>; + pick_gwg = pick_q4gsw_coop_gwg<8u, 8u>; + pick_lwg = pick_q4gsw_coop_lwg<8u, 8u>; + break; + default: + VK_THROW("add_q4gsw_linear_coop_kc_forced_node: non-coop kind"); + } + + graph.execute_nodes().emplace_back(new DynamicDispatchNode( + graph, + pick_shader, + pick_gwg, + pick_lwg, + {{output, vkapi::kWrite}, + {{fp_input, + dummy_transposed_input.vref, + packed_weight_kc, + packed_scales, + packed_bias}, + vkapi::kRead}}, + {graph.sizes_ubo(output), graph.sizes_ubo(fp_input)}, + {}, + {apply_bias, K_val, static_cast(group_size_val)}, + {weight_data, fp_input}, + resize_q4gsw_linear_node)); +} + +void add_q4gsw_linear_forced_node( + ComputeGraph& graph, + const ValueRef fp_input, + const ValueRef weight_data, + const ValueRef weight_scales_data, + const ValueRef group_size_ref, + const ValueRef bias_data, + const ValueRef output, + TestKernelKind kind) { + const vkapi::ScalarType in_dtype = graph.dtype_of(fp_input); + + const int64_t group_size_val = graph.extract_scalar(group_size_ref); + + std::vector weight_sizes = graph.sizes_of(weight_data); + const int64_t K = weight_sizes.at(1) * 2; + const uint32_t K_val = static_cast(K); + + const ValueRef packed_weight = prepack_q4_w_4x8_nc_buffer(graph, weight_data); + const ValueRef packed_scales = + prepack_q4_scales(graph, weight_scales_data, in_dtype); + + TmpTensor dummy_bias( + &graph, {}, graph.dtype_of(output), utils::kBuffer, utils::kWidthPacked); + ValueRef packed_bias = dummy_bias.vref; + uint32_t apply_bias = 0; + if (graph.val_is_not_none(bias_data)) { + packed_bias = + prepack_standard(graph, bias_data, utils::kBuffer, utils::kWidthPacked); + apply_bias = 1; + } + + const bool need_transpose = (kind == TestKernelKind::GEMM_TIN_W_4X8); + + std::vector in_sizes = graph.sizes_of(fp_input); + const uint32_t M_val = + utils::safe_downcast(utils::val_at(-2, in_sizes)); + const int64_t M4 = (static_cast(M_val) + 3) / 4; + + TmpTensor dummy_transposed_input( + &graph, {}, in_dtype, utils::kBuffer, utils::kWidthPacked); + TmpTensor real_transposed_input( + &graph, + {static_cast(K_val) * M4 * 4}, + in_dtype, + utils::kBuffer, + utils::kWidthPacked); + + ValueRef transposed_input_ref; + if (need_transpose) { + transposed_input_ref = real_transposed_input.vref; + add_transpose_cast_contig_to_vectorized_node( + graph, fp_input, transposed_input_ref); + } else { + transposed_input_ref = dummy_transposed_input.vref; + } + + using PickShaderFn = vkapi::ShaderInfo (*)( + ComputeGraph*, + const std::vector&, + const std::vector&); + using PickGwgFn = GlobalWorkGrid (*)( + ComputeGraph*, + const vkapi::ShaderInfo&, + const std::vector&, + const std::vector&); + using PickLwgFn = LocalWorkGroup (*)( + ComputeGraph*, + const vkapi::ShaderInfo&, + const GlobalWorkGrid&, + const std::vector&, + const std::vector&); + + PickShaderFn pick_shader = nullptr; + PickGwgFn pick_gwg = nullptr; + PickLwgFn pick_lwg = nullptr; + + switch (kind) { + case TestKernelKind::GEMM_W_4X8: + pick_shader = pick_forced_shader; + pick_gwg = pick_q4gsw_linear_gemm_gwg; + pick_lwg = pick_q4gsw_linear_gemm_lwg; + break; + case TestKernelKind::GEMV_W_4X8: + pick_shader = pick_forced_shader; + pick_gwg = pick_q4gsw_legacy_gemv_gwg; + pick_lwg = pick_q4gsw_legacy_gemv_lwg; + break; + case TestKernelKind::GEMM_TIN_W_4X8: + pick_shader = pick_forced_shader; + pick_gwg = pick_q4gsw_linear_tin_gemm_gwg; + pick_lwg = pick_q4gsw_linear_tin_gemm_lwg; + break; + case TestKernelKind::GEMV_W_4X8_NOSG: + pick_shader = pick_forced_shader; + pick_gwg = pick_q4gsw_legacy_gemv_gwg; + pick_lwg = pick_q4gsw_legacy_gemv_lwg; + break; + case TestKernelKind::PROD: + default: + VK_THROW("PROD kind must be dispatched via production entry points"); + } + + graph.execute_nodes().emplace_back(new DynamicDispatchNode( + graph, + pick_shader, + pick_gwg, + pick_lwg, + {{output, vkapi::kWrite}, + {{fp_input, + transposed_input_ref, + packed_weight, + packed_scales, + packed_bias}, + vkapi::kRead}}, + {graph.sizes_ubo(output), graph.sizes_ubo(fp_input)}, + {}, + {apply_bias, K_val, static_cast(group_size_val)}, + {weight_data, fp_input}, + resize_q4gsw_linear_node)); +} + +void add_fpa_q4gsw_linear_node( + ComputeGraph& graph, + const ValueRef fp_input, + const ValueRef weight_data, + const ValueRef weight_scales_data, + const ValueRef group_size_ref, + const ValueRef bias_data, + int32_t impl_selector_int, + bool is_gemv, + const ValueRef output) { + TestKernelKind kind = selector_to_kind(impl_selector_int, is_gemv); + + if (kind == TestKernelKind::PROD) { + std::vector q4gsw_linear_args = { + fp_input, + weight_data, + weight_scales_data, + group_size_ref, + bias_data, + output}; + VK_GET_OP_FN("et_vk.q4gsw_linear.default")(graph, q4gsw_linear_args); + return; + } + + if (kind == TestKernelKind::LEGACY) { + add_legacy_q4gsw_linear_node( + graph, + fp_input, + weight_data, + weight_scales_data, + group_size_ref, + bias_data, + output); + return; + } + + if (kind == TestKernelKind::GEMV_COOP_W_4X8_NC_BUFFER || + kind == TestKernelKind::GEMV_COOP_W_4X8_NC_BUFFER_G1W64 || + kind == TestKernelKind::GEMV_COOP_W_4X8_NC_BUFFER_G4W16 || + kind == TestKernelKind::GEMV_COOP_W_4X8_NC_BUFFER_G8W8) { + add_q4gsw_linear_coop_kc_forced_node( + graph, + fp_input, + weight_data, + weight_scales_data, + group_size_ref, + bias_data, + output, + kind); + return; + } + + add_q4gsw_linear_forced_node( + graph, + fp_input, + weight_data, + weight_scales_data, + group_size_ref, + bias_data, + output, + kind); +} + +} // namespace + +namespace shader_profiling { +namespace { +template +Json build_q4_route( + ComputeGraph& graph, + const std::vector& args, + const Json& options) { + if (!options.empty()) + throw std::invalid_argument("Q4 routes accept no options"); + if (args.size() != 6) + throw std::invalid_argument("Q4 route requires six canonical arguments"); + const auto* adapter = graph.context()->adapter_ptr(); + if constexpr (Selector == 1 && Gemv) { + if (!adapter->supports_required_subgroup_size_for_compute() || + adapter->min_subgroup_size() > 64 || + adapter->max_subgroup_size() < 64 || + !adapter->supports_subgroup_compute_basic() || + !adapter->supports_subgroup_compute_ballot() || + !adapter->supports_subgroup_compute_shuffle()) + throw UnsupportedRoute( + "gemv_sg requires supported subgroup64 basic/ballot/shuffle"); + } + if constexpr ((Selector == 1 || Selector == 2) && Gemv) { + if (adapter->max_compute_workgroup_invocations() < 256 || + adapter->max_compute_workgroup_size()[0] < 64 || + adapter->max_compute_workgroup_size()[1] < 4) + throw UnsupportedRoute("Legacy GEMV requires a 64x4 workgroup"); + } + if constexpr (Gemv) { + if (graph.sizes_of(args[0]).at(0) != 1) + throw UnsupportedRoute("GEMV route requires M=1"); + } + add_fpa_q4gsw_linear_node( + graph, + args[0], + args[1], + args[2], + args[3], + args[4], + Selector, + Gemv, + args[5]); + const char* name = Selector == 3 ? "legacy" + : Gemv ? (Selector == 1 ? "gemv_sg" + : Selector == 2 ? "gemv_nosg" + : Selector == 13 ? "gemv_coop" + : Selector == 14 ? "gemv_g1w64" + : Selector == 15 ? "gemv_g4w16" + : "gemv_g8w8") + : (Selector == 1 ? "gemm" : "gemm_tin"); + return {{"name", name}, {"options", options}}; +} +REGISTER_OPERATOR_ROUTES { + REGISTER_OPERATOR_ROUTE( + et_vk.q4gsw_linear.default, gemm, (build_q4_route<1, false>)); + REGISTER_OPERATOR_ROUTE( + et_vk.q4gsw_linear.default, gemm_tin, (build_q4_route<2, false>)); + REGISTER_OPERATOR_ROUTE( + et_vk.q4gsw_linear.default, legacy, (build_q4_route<3, false>)); + REGISTER_OPERATOR_ROUTE( + et_vk.q4gsw_linear.default, gemv_sg, (build_q4_route<1, true>)); + REGISTER_OPERATOR_ROUTE( + et_vk.q4gsw_linear.default, gemv_nosg, (build_q4_route<2, true>)); + REGISTER_OPERATOR_ROUTE( + et_vk.q4gsw_linear.default, gemv_coop, (build_q4_route<13, true>)); + REGISTER_OPERATOR_ROUTE( + et_vk.q4gsw_linear.default, gemv_g1w64, (build_q4_route<14, true>)); + REGISTER_OPERATOR_ROUTE( + et_vk.q4gsw_linear.default, gemv_g4w16, (build_q4_route<15, true>)); + REGISTER_OPERATOR_ROUTE( + et_vk.q4gsw_linear.default, gemv_g8w8, (build_q4_route<16, true>)); +} +} // namespace +} // namespace shader_profiling +} // namespace vkcompute diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/cases/coopmat_correctness.json b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/cases/coopmat_correctness.json new file mode 100644 index 00000000000..0934e94faf1 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/cases/coopmat_correctness.json @@ -0,0 +1,1466 @@ +[ + { + "name": "q4gsw_linear coopmat_correctness [64, 128, 64, 64]", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 5, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.5, + "rtol": 0.05 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "repeat": [0.5, 0.625, 0.75, 0.875, 1.0, 1.125, 1.25, 1.375] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [153, 170, 187, 204, 221, 238] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 64], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 64 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [64, 64], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear coopmat_correctness [64, 128, 256, 64]", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 5, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.5, + "rtol": 0.05 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "repeat": [0.5, 0.625, 0.75, 0.875, 1.0, 1.125, 1.25, 1.375] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [153, 170, 187, 204, 221, 238] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 256], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 64 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [64, 256], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear coopmat_correctness [128, 256, 128, 64]", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 5, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.5, + "rtol": 0.05 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [128, 256], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "repeat": [0.5, 0.625, 0.75, 0.875, 1.0, 1.125, 1.25, 1.375] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 128], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [153, 170, 187, 204, 221, 238] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 128], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 64 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [128, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear coopmat_correctness [128, 128, 128, 64]", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 5, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.5, + "rtol": 0.05 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [128, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "repeat": [0.5, 0.625, 0.75, 0.875, 1.0, 1.125, 1.25, 1.375] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [153, 170, 187, 204, 221, 238] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 128], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 64 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [128, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear coopmat_correctness [128, 128, 256, 64]", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 5, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.5, + "rtol": 0.05 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [128, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "repeat": [0.5, 0.625, 0.75, 0.875, 1.0, 1.125, 1.25, 1.375] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [153, 170, 187, 204, 221, 238] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 256], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 64 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [128, 256], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear coopmat_correctness [256, 256, 256, 64]", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 5, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.5, + "rtol": 0.05 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [256, 256], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "repeat": [0.5, 0.625, 0.75, 0.875, 1.0, 1.125, 1.25, 1.375] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 128], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [153, 170, 187, 204, 221, 238] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 256], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 64 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [256, 256], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear coopmat_correctness [256, 128, 128, 64]", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 5, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.5, + "rtol": 0.05 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [256, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "repeat": [0.5, 0.625, 0.75, 0.875, 1.0, 1.125, 1.25, 1.375] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [153, 170, 187, 204, 221, 238] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 128], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 64 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [256, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear coopmat_correctness [256, 128, 64, 64]", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 5, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.5, + "rtol": 0.05 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [256, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "repeat": [0.5, 0.625, 0.75, 0.875, 1.0, 1.125, 1.25, 1.375] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [153, 170, 187, 204, 221, 238] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 64], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 64 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [256, 64], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear coopmat_correctness [64, 128, 64, 64] (2)", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 5, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.5, + "rtol": 0.05 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "repeat": [0.5, 0.625, 0.75, 0.875, 1.0, 1.125, 1.25, 1.375] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 64], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0.0625] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 64], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 0] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [153, 170, 187, 204, 221, 238] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [2, 64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 64 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 64], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 64 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [64, 64], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear coopmat_correctness [64, 128, 256, 64] (2)", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 5, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.5, + "rtol": 0.05 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "repeat": [0.5, 0.625, 0.75, 0.875, 1.0, 1.125, 1.25, 1.375] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 64], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0.0625] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 64], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 0] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [153, 170, 187, 204, 221, 238] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [2, 256], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 64 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 256], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 64 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [64, 256], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear coopmat_correctness [128, 256, 128, 64] (2)", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 5, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.5, + "rtol": 0.05 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [128, 256], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "repeat": [0.5, 0.625, 0.75, 0.875, 1.0, 1.125, 1.25, 1.375] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 128], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0.0625] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 128], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 0] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 128], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [153, 170, 187, 204, 221, 238] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4, 128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 64 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 128], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 64 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [128, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear coopmat_correctness [128, 128, 128, 64] (2)", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 5, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.5, + "rtol": 0.05 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [128, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "repeat": [0.5, 0.625, 0.75, 0.875, 1.0, 1.125, 1.25, 1.375] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 128], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0.0625] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 128], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 0] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [153, 170, 187, 204, 221, 238] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [2, 128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 64 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 128], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 64 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [128, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear coopmat_correctness [128, 128, 256, 64] (2)", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 5, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.5, + "rtol": 0.05 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [128, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "repeat": [0.5, 0.625, 0.75, 0.875, 1.0, 1.125, 1.25, 1.375] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 128], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0.0625] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 128], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 0] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [153, 170, 187, 204, 221, 238] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [2, 256], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 64 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 256], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 64 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [128, 256], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear coopmat_correctness [256, 256, 256, 64] (2)", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 5, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.5, + "rtol": 0.05 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [256, 256], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "repeat": [0.5, 0.625, 0.75, 0.875, 1.0, 1.125, 1.25, 1.375] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 256], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0.0625] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 256], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 0] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 128], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [153, 170, 187, 204, 221, 238] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4, 256], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 64 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 256], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 64 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [256, 256], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear coopmat_correctness [256, 128, 128, 64] (2)", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 5, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.5, + "rtol": 0.05 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [256, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "repeat": [0.5, 0.625, 0.75, 0.875, 1.0, 1.125, 1.25, 1.375] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 256], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0.0625] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 256], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 0] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [153, 170, 187, 204, 221, 238] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [2, 128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 64 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 128], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 64 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [256, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear coopmat_correctness [256, 128, 64, 64] (2)", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 5, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.5, + "rtol": 0.05 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [256, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "repeat": [0.5, 0.625, 0.75, 0.875, 1.0, 1.125, 1.25, 1.375] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 256], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [0.0625] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 256], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 0] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "repeat": [153, 170, 187, 204, 221, 238] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [2, 64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 64 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 64], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 64 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [256, 64], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/cases/coopmat_performance.json b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/cases/coopmat_performance.json new file mode 100644 index 00000000000..31f8f4feb24 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/cases/coopmat_performance.json @@ -0,0 +1,734 @@ +[ + { + "name": "q4gsw_linear coopmat_performance [1024, 4096, 4096, 128]", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 5, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1024, 4096], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [4096, 2048], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [32, 4096], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 128 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1024, 4096], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear coopmat_performance [1024, 4096, 1024, 128]", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 5, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1024, 4096], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [1024, 2048], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [32, 1024], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 128 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1024, 1024], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear coopmat_performance [1024, 4096, 14336, 128]", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 5, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1024, 4096], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [14336, 2048], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [32, 14336], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 128 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1024, 14336], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear coopmat_performance [1024, 14336, 4096, 128]", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 5, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1024, 14336], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [4096, 7168], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [112, 4096], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 128 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1024, 4096], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear coopmat_performance [1024, 4096, 4096, 128] (2)", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 5, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1024, 4096], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 1024], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 1024], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [4096, 2048], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32, 4096], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 128 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [32, 4096], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 128 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1024, 4096], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear coopmat_performance [1024, 4096, 1024, 128] (2)", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 5, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1024, 4096], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 1024], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 1024], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [1024, 2048], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32, 1024], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 128 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [32, 1024], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 128 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1024, 1024], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear coopmat_performance [1024, 4096, 14336, 128] (2)", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 5, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1024, 4096], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 1024], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 1024], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [14336, 2048], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32, 14336], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 128 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [32, 14336], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 128 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1024, 14336], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear coopmat_performance [1024, 14336, 4096, 128] (2)", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 5, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1024, 14336], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 1024], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 1024], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [4096, 7168], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [112, 4096], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 128 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [112, 4096], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 128 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1024, 4096], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/cases/correctness.json b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/cases/correctness.json new file mode 100644 index 00000000000..5d452a04a64 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/cases/correctness.json @@ -0,0 +1,6290 @@ +[ + { + "name": "q4gsw_linear correctness [4, 64, 32, 16] float32 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 4], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4, 32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 16 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 16 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [4, 32], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [32, 64, 32, 16] float32 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 32], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4, 32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 16 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 16 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [32, 32], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [4, 128, 64, 32] float32 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 4], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4, 64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 32 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [4, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [32, 128, 64, 32] float32 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 32], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4, 64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 32 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [32, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [256, 128, 128, 32] float32 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [256, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 256], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 256], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4, 128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 32 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [256, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness default M2 biasFalse float32 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [2, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 2], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 2], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [2, 32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 32 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [2, 32], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness default M3 biasFalse float32 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [3, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 3], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 3], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [2, 32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 32 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [3, 32], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness default M5 biasFalse float32 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [5, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 5], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 5], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [2, 32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 32 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [5, 32], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [4, 256, 128, 64] float32 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4, 256], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 4], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 128], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4, 128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 64 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 64 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [4, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [32, 256, 128, 64] float32 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 256], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 32], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 128], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4, 128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 64 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 64 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [32, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [64, 64, 64, 64] float32 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 64], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [1, 64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 64 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [1, 64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 64 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [64, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [64, 128, 64, 64] float32 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 64], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [2, 64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 64 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 64 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [64, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [64, 256, 128, 128] float32 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 256], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 64], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 128], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [2, 128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 128 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 128 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [64, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [4, 64, 32, 16] float16 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4, 64], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 4], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 4], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4, 32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 16 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 32], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 16 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [4, 32], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [32, 64, 32, 16] float16 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 64], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 32], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 32], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4, 32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 16 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 32], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 16 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [32, 32], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [64, 64, 64, 64] float16 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 64], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 64], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 64], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [1, 64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 64 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [1, 64], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 64 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [64, 64], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness default M2 biasFalse float16 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [2, 64], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 2], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 2], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [2, 32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 32 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 32], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [2, 32], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness default M3 biasFalse float16 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [3, 64], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 3], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 3], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [2, 32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 32 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 32], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [3, 32], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness default M5 biasFalse float16 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [5, 64], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 5], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 5], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [2, 32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 32 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 32], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [5, 32], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [4, 128, 64, 32] float16 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.1313708498984762, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 4], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 4], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4, 64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 32 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 64], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [4, 64], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [32, 128, 64, 32] float16 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.1313708498984762, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 32], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 32], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4, 64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 32 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 64], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [32, 64], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [256, 128, 128, 32] float16 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.1313708498984762, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [256, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 256], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 256], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4, 128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 32 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 128], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [256, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [64, 128, 64, 64] float16 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.1313708498984762, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 64], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 64], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [2, 64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 64 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 64], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 64 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [64, 64], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [4, 256, 128, 64] float16 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.6, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4, 256], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 4], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 4], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 128], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4, 128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 64 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 128], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 64 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [4, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [32, 256, 128, 64] float16 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.6, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 256], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 32], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 32], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 128], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4, 128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 64 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 128], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 64 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [32, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [64, 256, 128, 128] float16 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.6, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 256], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 64], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 64], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 128], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [2, 128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 128 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 128], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 128 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [64, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [4, 64, 32, 16] (3)", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 16 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [4, 32], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [32, 64, 32, 16] (3)", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 16 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [32, 32], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [4, 128, 64, 32] (3)", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [4, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [32, 128, 64, 32] (3)", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [32, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [256, 128, 128, 32] (3)", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [256, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [256, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [4, 256, 128, 64] (3)", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4, 256], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 128], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 64 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [4, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [32, 256, 128, 64] (3)", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 256], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 128], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 64 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [32, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [64, 64, 64, 64] (3)", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [1, 64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 64 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [64, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [64, 128, 64, 64] (3)", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 64 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [64, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [64, 256, 128, 128] (3)", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 256], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 128], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 128 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [64, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [4, 64, 32, 16] float16 width (36)", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4, 64], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 32], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 16 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [4, 32], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [32, 64, 32, 16] float16 width (37)", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 64], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 32], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 16 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [32, 32], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [64, 64, 64, 64] float16 width (38)", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 64], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [1, 64], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 64 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [64, 64], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [4, 128, 64, 32] float16 width (39)", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.1313708498984762, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 64], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [4, 64], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [32, 128, 64, 32] float16 width (40)", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.1313708498984762, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 64], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [32, 64], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [256, 128, 128, 32] float16 width (41)", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.1313708498984762, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [256, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 128], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [256, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [64, 128, 64, 64] float16 width (42)", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.1313708498984762, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 64], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 64 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [64, 64], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [4, 256, 128, 64] float16 width (43)", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.6, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4, 256], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 128], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 128], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 64 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [4, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [32, 256, 128, 64] float16 width (44)", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.6, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 256], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 128], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 128], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 64 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [32, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [64, 256, 128, 128] float16 width (45)", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.6, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 256], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 128], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 128], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 128 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [64, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [4, 64, 32, 16, True] float32 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 4], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4, 32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 16 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 16 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4, 32], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [4, 128, 64, 32, True] float32 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 4], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4, 64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 32 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [32, 128, 64, 32, True] float32 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 32], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4, 64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 32 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [32, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness default M2 biasTrue float32 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [2, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 2], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 2], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [2, 32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 32 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [2, 32], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness default M3 biasTrue float32 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [3, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 3], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 3], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [2, 32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 32 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [3, 32], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness default M5 biasTrue float32 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [5, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 5], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 5], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [2, 32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 32 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [5, 32], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [4, 64, 32, 16, True] float16 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4, 64], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 4], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 4], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4, 32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 16 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 32], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 16 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4, 32], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness default M2 biasTrue float16 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [2, 64], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 2], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 2], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [2, 32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 32 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 32], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [2, 32], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness default M3 biasTrue float16 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [3, 64], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 3], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 3], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [2, 32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 32 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 32], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [3, 32], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness default M5 biasTrue float16 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [5, 64], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 5], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 5], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [2, 32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 32 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 32], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [5, 32], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [4, 128, 64, 32, True] float16 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.1313708498984762, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 4], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 4], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4, 64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 32 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 64], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4, 64], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [32, 128, 64, 32, True] float16 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.1313708498984762, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 32], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 32], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4, 64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 32 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 64], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [32, 64], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [4, 64, 32, 16, True] (3)", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 16 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4, 32], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [4, 128, 64, 32, True] (3)", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [32, 128, 64, 32, True] (3)", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [32, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [4, 64, 32, 16, True] float16 width (61)", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4, 64], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 32], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 16 + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4, 32], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [4, 128, 64, 32, True] float16 width (62)", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.1313708498984762, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 64], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4, 64], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear correctness [32, 128, 64, 32, True] float16 width (63)", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.1313708498984762, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 64], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [32, 64], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/cases/debug.json b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/cases/debug.json new file mode 100644 index 00000000000..93ea2bda3f2 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/cases/debug.json @@ -0,0 +1,124 @@ +[ + { + "name": "q4gsw_linear debug", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [8, 16], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 8], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 8], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [16, 8], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [2, 16], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 8 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 8 + }, + { + "tensor": { + "name": "bias", + "sizes": [16], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [8, 16], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/cases/fpa_correctness.json b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/cases/fpa_correctness.json new file mode 100644 index 00000000000..db32654242e --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/cases/fpa_correctness.json @@ -0,0 +1,1458 @@ +[ + { + "name": "q4gsw_linear fpa_correctness M1 K128 N128 float32", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "gemv_nosg", "gemv_coop"], + "v_STORAGE": ["buffer", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_correctness M1 K128 N256 float32", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["gemv_g1w64", "gemv_g4w16", "gemv_g8w8"], + "v_STORAGE": ["buffer", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 256], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 256], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_correctness M32 K128 N12 float32", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "gemm", "gemm_tin"], + "v_STORAGE": ["buffer", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [12, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 12], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [32, 12], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_correctness M32 K128 N20 float32", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "gemm", "gemm_tin"], + "v_STORAGE": ["buffer", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [20, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 20], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [32, 20], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_correctness M32 K128 N128 float32 texture3d selector0", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.001 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [32, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_correctness M31 K128 N128 float32 texture3d selector0", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.001 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [31, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [31, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_correctness M33 K128 N128 float32 texture3d selector0", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.001 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [33, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [33, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_correctness M1 K128 N128 float16", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.4, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "gemv_nosg", "gemv_coop"], + "v_STORAGE": ["buffer", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 128], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_correctness M1 K128 N256 float16", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.4, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["gemv_g1w64", "gemv_g4w16", "gemv_g8w8"], + "v_STORAGE": ["buffer", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 256], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 256], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_correctness M32 K128 N12 float16", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.4, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "gemm", "gemm_tin"], + "v_STORAGE": ["buffer", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [12, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 12], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [32, 12], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_correctness M32 K128 N20 float16", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.4, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "gemm", "gemm_tin"], + "v_STORAGE": ["buffer", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [20, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 20], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [32, 20], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_correctness M32 K128 N128 float16 texture3d selector0", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.4, + "rtol": 0.001 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 128], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [32, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_correctness M31 K128 N128 float16 texture3d selector0", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.4, + "rtol": 0.001 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [31, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 128], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [31, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_correctness M33 K128 N128 float16 texture3d selector0", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.4, + "rtol": 0.001 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [33, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 64], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [4, 128], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [33, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_correctness M1 K256 N256 float32", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.2, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "gemv_nosg", "gemv_coop"], + "v_STORAGE": ["buffer", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 256], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 128], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [8, 256], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 256], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_correctness M1 K256 N128 float32", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.2, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["gemv_g1w64", "gemv_g4w16", "gemv_g8w8"], + "v_STORAGE": ["buffer", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 256], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 128], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [8, 128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_correctness M1 K64 N128 float16", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.2, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["gemv_g1w64", "gemv_g4w16", "gemv_g8w8"], + "v_STORAGE": ["buffer", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 128], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_correctness M1 K256 N256 float16", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.8, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "gemv_nosg", "gemv_coop"], + "v_STORAGE": ["buffer", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 256], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 128], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [8, 256], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 256], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_correctness M1 K256 N128 float16", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.8, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["gemv_g1w64", "gemv_g4w16", "gemv_g8w8"], + "v_STORAGE": ["buffer", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 256], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 128], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [8, 128], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 128], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_correctness M1 K64 N128 float32", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.05, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["gemv_g1w64", "gemv_g4w16", "gemv_g8w8"], + "v_STORAGE": ["buffer", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 32], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [2, 128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/cases/fpa_performance.json b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/cases/fpa_performance.json new file mode 100644 index 00000000000..3607514c87a --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/cases/fpa_performance.json @@ -0,0 +1,1897 @@ +[ + { + "name": "q4gsw_linear fpa_performance M1 K1024 N2048 float32", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.8, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "gemv_sg", "gemv_nosg", "legacy", "gemv_coop"], + "v_STORAGE": ["buffer", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 1024], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [2048, 512], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [32, 2048], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 2048], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_performance M32 K1024 N2048 float32", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.8, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "gemm", "gemm_tin", "legacy"], + "v_STORAGE": ["buffer", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 1024], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [2048, 512], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [32, 2048], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [32, 2048], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_performance M128 K1024 N2048 float32", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.8, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "gemm", "gemm_tin", "legacy"], + "v_STORAGE": ["buffer", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [128, 1024], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [2048, 512], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [32, 2048], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [128, 2048], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_performance M256 K1024 N2048 float32", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.8, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "gemm", "gemm_tin", "legacy"], + "v_STORAGE": ["buffer", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [256, 1024], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [2048, 512], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [32, 2048], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [256, 2048], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_performance M1 K1024 N2048 float16 buffer", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 3.2, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "gemv_sg", "gemv_nosg", "legacy", "gemv_coop"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 1024], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [2048, 512], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [32, 2048], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 2048], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_performance M1 K1024 N2048 float16 texture3d", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 3.2, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "gemv_sg", "gemv_nosg", "legacy", "gemv_coop"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 1024], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [2048, 512], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [32, 2048], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 2048], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 1024], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [2048, 512], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [32, 2048], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 2048], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_performance M1 K1024 N1024 float16 texture3d", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 3.2, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "gemv_sg", "gemv_nosg", "legacy", "gemv_coop"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 1024], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [1024, 512], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [32, 1024], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 1024], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_performance M1 K1024 N3072 float16 texture3d", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 3.2, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "gemv_sg", "gemv_nosg", "legacy", "gemv_coop"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 1024], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [3072, 512], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [32, 3072], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 3072], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_performance M32 K1024 N2048 float16", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 3.2, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "gemm", "gemm_tin", "legacy"], + "v_STORAGE": ["buffer", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 1024], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [2048, 512], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [32, 2048], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [32, 2048], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_performance M128 K1024 N2048 float16", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 3.2, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "gemm", "gemm_tin", "legacy"], + "v_STORAGE": ["buffer", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [128, 1024], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [2048, 512], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [32, 2048], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [128, 2048], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_performance M256 K1024 N2048 float16", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 3.2, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "gemm", "gemm_tin", "legacy"], + "v_STORAGE": ["buffer", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [256, 1024], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [2048, 512], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [32, 2048], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [256, 2048], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_performance M1 K4096 N4096 float32", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 3.2, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "gemv_sg", "gemv_nosg", "legacy", "gemv_coop"], + "v_STORAGE": ["buffer", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 4096], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [4096, 2048], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [128, 4096], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 4096], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_performance M32 K4096 N4096 float32", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 3.2, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "gemm", "gemm_tin", "legacy"], + "v_STORAGE": ["buffer", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 4096], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [4096, 2048], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [128, 4096], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [32, 4096], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_performance M128 K4096 N4096 float32", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 3.2, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "gemm", "gemm_tin", "legacy"], + "v_STORAGE": ["buffer", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [128, 4096], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [4096, 2048], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [128, 4096], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [128, 4096], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_performance M256 K4096 N4096 float32", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 3.2, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "gemm", "gemm_tin", "legacy"], + "v_STORAGE": ["buffer", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [256, 4096], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [4096, 2048], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [128, 4096], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [256, 4096], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_performance M1 K4096 N4096 float16", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 12.8, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "gemv_sg", "gemv_nosg", "legacy", "gemv_coop"], + "v_STORAGE": ["buffer", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 4096], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [4096, 2048], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [128, 4096], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 4096], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_performance M32 K4096 N4096 float16", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 12.8, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "gemm", "gemm_tin", "legacy"], + "v_STORAGE": ["buffer", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 4096], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [4096, 2048], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [128, 4096], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [32, 4096], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_performance M128 K4096 N4096 float16", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 12.8, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "gemm", "gemm_tin", "legacy"], + "v_STORAGE": ["buffer", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [128, 4096], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [4096, 2048], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [128, 4096], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [128, 4096], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_performance M256 K4096 N4096 float16", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 12.8, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "gemm", "gemm_tin", "legacy"], + "v_STORAGE": ["buffer", "texture3d"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [256, 4096], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [4096, 2048], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [128, 4096], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [256, 4096], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_performance M1 K2048 N512 float16 texture3d", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 6.4, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "gemv_sg", "gemv_nosg", "legacy", "gemv_coop"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 2048], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [512, 1024], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [64, 512], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 512], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_performance M1 K2048 N2048 float16 texture3d", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 6.4, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "gemv_sg", "gemv_nosg", "legacy", "gemv_coop"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 2048], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [2048, 1024], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [64, 2048], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 2048], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_performance M1 K2048 N8192 float16 texture3d", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 6.4, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "gemv_sg", "gemv_nosg", "legacy", "gemv_coop"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 2048], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [8192, 1024], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [64, 8192], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 8192], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_performance M1 K8192 N2048 float16 texture3d", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 25.6, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "gemv_sg", "gemv_nosg", "legacy", "gemv_coop"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 8192], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [2048, 4096], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [256, 2048], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 2048], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_performance M1 K3072 N1024 float16 texture3d", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 9.600000000000001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "gemv_sg", "gemv_nosg", "legacy", "gemv_coop"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 3072], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [1024, 1536], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [96, 1024], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 1024], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear fpa_performance M1 K2048 N2048 float32", + "operator": "et_vk.q4gsw_linear.default", + "target_ms": 100, + "warmup_runs": 3, + "benchmark_runs": 10, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.6, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "gemv_sg", "gemv_nosg", "legacy", "gemv_coop"], + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 2048], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [2048, 1024], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [64, 2048], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 32 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1, 2048], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/cases/performance.json b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/cases/performance.json new file mode 100644 index 00000000000..3a23488efde --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/cases/performance.json @@ -0,0 +1,734 @@ +[ + { + "name": "q4gsw_linear performance [128, 2048, 2048, 128] float32 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [128, 2048], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 128], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [2048, 1024], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [16, 2048], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 128 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [16, 2048], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 128 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [128, 2048], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear performance [1024, 4096, 4096, 128] float32 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1024, 4096], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 1024], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 1024], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [4096, 2048], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32, 4096], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 128 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [32, 4096], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 128 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1024, 4096], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear performance [128, 2048, 2048, 128] float16 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 4.525483399593905, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [128, 2048], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 128], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 128], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [2048, 1024], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [16, 2048], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 128 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [16, 2048], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 128 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [128, 2048], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear performance [1024, 4096, 4096, 128] float16 width", + "operator": "et_vk.linear_dq8ca_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 6.4, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1024, 4096], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "input_scale", + "sizes": [1, 1024], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "tensor": { + "name": "input_zero_point", + "sizes": [1, 1024], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [4096, 2048], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32, 4096], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "q4_group_sum": { + "tensor": "weight", + "group_size": 128 + } + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [32, 4096], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 128 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1024, 4096], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear performance [128, 2048, 2048, 128] (3)", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [128, 2048], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [2048, 1024], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [16, 2048], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 128 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [128, 2048], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear performance [1024, 4096, 4096, 128] (3)", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1024, 4096], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [4096, 2048], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [32, 4096], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 128 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1024, 4096], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear performance [128, 2048, 2048, 128] float16 width (6)", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 4.525483399593905, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [128, 2048], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [2048, 1024], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [16, 2048], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 128 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [128, 2048], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q4gsw_linear performance [1024, 4096, 4096, 128] float16 width (7)", + "operator": "et_vk.linear_q4gsw.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 6.4, + "rtol": 0.1 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1024, 4096], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_int": [-10, 10] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [4096, 2048], + "dtype": "uint8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [0, 255] + } + } + }, + { + "tensor": { + "name": "scales", + "sizes": [32, 4096], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.005, 0.015] + } + } + }, + { + "int": 128 + }, + null, + { + "tensor": { + "name": "output", + "sizes": [1024, 4096], + "dtype": "float16", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/glsl/q4gsw_linear_gemv__w_4x8.glsl b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/glsl/q4gsw_linear_gemv__w_4x8.glsl new file mode 100644 index 00000000000..e3c6ccdfba7 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/glsl/q4gsw_linear_gemv__w_4x8.glsl @@ -0,0 +1,371 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +// q4gsw linear GEMV kernel — row-pair broadcast dequant-accumulate over the +// shared w_4x8 weight prepack. +// +// Shader naming convention: +// q4gsw_linear_gemv__w_4x8_[_nosg] +// ^^^^^^^^^^^^^^^^^ ^^^^^ ^^^^^^^ +// op base (gemv) tile tile arrangement (nc or kc) +// +// Weight binding: +// The shared pack_q4_linear_weight__w_4x8 shader writes a W_4X8 block-packed uvec2 buffer +// where the uvec2 at logical tile index [k4, n4] lives at the 2 consecutive +// uint slots: +// t_q4_weights[2 * tile_idx + 0] = .x (row pair {N0, N1}) +// t_q4_weights[2 * tile_idx + 1] = .y (row pair {N2, N3}) +// Read as a scalar uint buffer, the uint at +// word_idx = 2 * tile_idx + half (half in {0, 1}) +// is one N-row-pair's 4 K-step payload. With n2 = 2 * n4 + half and +// k_slot = k4, under WEIGHT_TILE_CONTIG_DIM=0: +// word_idx = k_slot * (N/2) + n2 +// which is the index formula used by the per-row-pair loop here. +// +// Interleaved (dp4a-style) byte-pair layout: each uvec2 lane's 4 bytes hold +// 4 K-consecutive positions for a pair of N rows. As a scalar uint, byte b +// of w_pack = (N_even, K=b) | (N_odd, K=b) << 4 — the low nibble per byte +// is the even-N row, the high nibble is odd-N. This byte packing is the +// natural memory split for the paired-row dequant below and lets the same +// shader body be repurposed later for int8/int4 integer matmul that +// operates directly on byte-interleaved nibble pairs. +// +// Scale binding: +// Uses the production dtype-matched scale bytes but reinterprets them as a +// gvec2 (vec2 / f16vec2) array. The scale prepack emits vec4 bytes indexed +// as t_scales[group_i * N4 + n4] where each vec4 holds 4 N-row scales. The +// same byte layout is addressable as vec2 with index +// vec2_idx = 2 * (group_i * N4 + n4) + half = group_i * N2 + n2 +// since each vec4 = 2 consecutive vec2 slots (low half = rows {2*n4, 2*n4+1}, +// high half = rows {2*n4+2, 2*n4+3}). Binding as vec2 halves the scale load +// byte volume and eliminates the 2 wasted components per load. +// +// WG layout: SUBGROUP_SIZE=64, NUM_SUBGROUPS=4. y-dim splits K-blocks across waves. +// Each thread owns one row-pair (n2) and writes two output floats. + +#version 450 core + +${define_required_extensions(IO_STORAGE, DTYPE)} +${define_required_extensions("buffer", DTYPE)} +#extension GL_EXT_shader_explicit_arithmetic_types_float16 : require +#extension GL_EXT_control_flow_attributes : require +$if USE_SUBGROUP_BROADCAST: + #extension GL_KHR_shader_subgroup_basic : require + #extension GL_KHR_shader_subgroup_ballot : require + #extension GL_KHR_shader_subgroup_shuffle : require + +#define PRECISION ${PRECISION} + +#define T ${texel_load_component_type(DTYPE, "buffer")} + +$if IO_STORAGE == "buffer": + #define IO_BUFFER + +#define NUM_SUBGROUPS ${NUM_SUBGROUPS} +#define SUBGROUP_SIZE ${SUBGROUP_SIZE} +// Workgroup x-dim size — used for shared-mem indexing in the inter-wave +// reduction. Chosen to match LWG.x set by the host (kGemvSubgroupSize=64). +// In the sg variant, lanes happen to align 1:1 with x-threads when +// subgroupSize >= LWG_X_SIZE; in the nosg variant, x-thread index alone +// addresses shared-mem slots so any subgroup width is safe. +#define LWG_X_SIZE ${LWG_X_SIZE} +// Number of K elements processed per outer k-loop iteration. Format-level +// constant — each iteration loads 8 vec4 of activations (= 32 K-vals) and 8 +// uint32 weight packs (4 hi + 4 lo, each holding 4 K-vals × 2 N-rows). Distinct +// from `group_size` (the quantization group): blocks_per_group = group_size / +// K_PER_STEP tells how many consecutive K-blocks share one scale pair. +#define K_PER_STEP 32 + +#define WEIGHT_TILE_CONTIG_DIM ${WEIGHT_TILE_CONTIG_DIM} + +layout(std430) buffer; + +// Unified 6-binding layout shared across q4gsw_linear shaders so a single +// DynamicDispatchNode with pick_shader_fn can switch between GEMM and GEMV +// kernels. This shader reads t_fp_input (the raw activation). The +// t_transposed_input binding is declared to preserve slot order but is never +// referenced here — the driver compiles it out to zero runtime cost; only +// the descriptor slot is allocated. +// +// Output: [1, N] scalar DTYPE buffer OR 1x1xN/4 texture3d. +// is_scalar_array is only meaningful for buffer storage; ignored for texture. +${layout_declare_tensor(B, "w", "t_output", DTYPE, IO_STORAGE, is_scalar_array=True)} +// Activations: [1, K] vec4-packed. +${layout_declare_tensor(B, "r", "t_fp_input", DTYPE, IO_STORAGE, is_scalar_array=False)} +// Unused transposed input — declared only so this shader shares the +// descriptor set layout with the tin GEMM shader. +${layout_declare_tensor(B, "r", "t_transposed_input", DTYPE, "buffer", is_scalar_array=False)} +// Weight: same uvec2 W_4X8 block-packed buffer produced by pack_q4_linear_weight__w_4x8, +// bound here as a scalar uint array so the per-row-pair index math can address +// individual uint slots directly. See header comment for byte-layout proof. +${layout_declare_tensor(B, "r", "t_q4_weights", "int", "buffer")} +// Scales: dtype-matched gvec2 reinterpret of the GEMM vec4 scale prepack. +// Indexed as t_scales[group_idx * N2 + n2]. +${layout_declare_tensor(B, "r", "t_scales", DTYPE, "buffer", is_scalar_array=False, vec_size=2)} +// Bias: [N] DTYPE buffer. +${layout_declare_tensor(B, "r", "t_bias", DTYPE, "buffer", is_scalar_array=True)} + +${layout_declare_ubo(B, "ivec4", "output_sizes")} +${layout_declare_ubo(B, "ivec4", "input_sizes")} + +layout(local_size_x_id = 0, local_size_y_id = 1, local_size_z_id = 2) in; + +${layout_declare_spec_const(C, "int", "apply_bias", "0")} +// `K` is declared only to keep the spec-constant layout aligned with the GEMM +// shaders so both variants can share a single DynamicDispatchNode with a +// runtime shader picker. It is not referenced in the GEMV body — the local +// `K` (derived from `input_sizes.x`) shadows it inside main(). +${layout_declare_spec_const(C, "int", "K", "1024")} +// Quantization group size in elements. `blocks_per_group` (the original GEMV +// spec constant) is recomputed from `group_size` below since K_PER_STEP = 32. +${layout_declare_spec_const(C, "int", "group_size", "32")} + +// Inter-wave reduction buffer (NUM_SUBGROUPS - 1 slabs of LWG_X_SIZE vec2). +// Slots are addressed by x-thread index, not subgroup lane index — sized to +// LWG.x so the shader is portable across subgroup widths. +shared vec2 partial_sums[LWG_X_SIZE * (NUM_SUBGROUPS - 1)]; + +$if not USE_SUBGROUP_BROADCAST: + // Used by the texture-storage write path to swap acc with the n2-XOR-1 + // partner thread. Replaces subgroupShuffleXor in the nosg variant. + shared vec2 nosg_n2_partner[LWG_X_SIZE]; + +// Load a vec4 of activations from input at (vec4 index) idx. +vec4 load_input_vec4(const int idx) { +#ifdef IO_BUFFER + return vec4(t_fp_input[idx]); +#else + return vec4(texelFetch(t_fp_input, ivec3(idx, 0, 0), 0)); +#endif +} + +// Load 2 scales for (n2, group) directly as a gvec2. +// The scale prepack bytes are reinterpreted as gvec2[group_idx * N2 + n2] +// where gvec2 is vec2 (fp32) or f16vec2 (fp16). The vec2(...) cast is a no-op +// for fp32 and an f16 -> f32 widening for fp16. +vec2 load_scale_pair(const int n2, const int group_idx, const int N2) { + return vec2(t_scales[group_idx * N2 + n2]); +} + +void main() { + $if USE_SUBGROUP_BROADCAST: + // sg path: lane_id == subgroup invocation; relies on subgroupSize == LWG.x + // (=64) so subgroup lane and x-thread coincide for shared-mem indexing. + const uint lane_id = gl_SubgroupInvocationID; + $else: + // nosg path: lane_id == x-thread within workgroup; portable across any + // subgroup width since shared-mem slots are addressed purely by LWG.x. + const uint lane_id = gl_LocalInvocationID.x; + const int k_wave_id = int(gl_LocalInvocationID.y); + const int n2 = int(gl_GlobalInvocationID.x); + + const int N = output_sizes.x; + const int K = input_sizes.x; + const int N2 = N / 2; + const int num_steps = K / K_PER_STEP; + // Derived from the shared `group_size` spec constant. Each K_PER_STEP (=32) + // K-block is one "block"; blocks_per_group tells how many consecutive blocks + // share a single scale pair along K. + const int blocks_per_group = group_size / K_PER_STEP; + + // Words per K-block in the weight buffer. One K-block covers K_PER_STEP K-vals + // = K_PER_STEP/4 k4 slices, and each k4 slice is N2 word-pairs. So + // words_per_k_block = (K_PER_STEP / 4) * N2. + // `k * K_BLOCK_STRIDE_W` gives the absolute word offset to the start of + // K-block `k`. + const int K_BLOCK_STRIDE_W = (K_PER_STEP / 4) * N2; + + if (n2 >= N2) { + return; + } + + vec2 acc = vec2(0.0); + + // Loop over k-blocks, waves split k-blocks (k_wave_id, k_wave_id+NUM_SUBGROUPS, ...). + for (int k = k_wave_id; k < num_steps; k += NUM_SUBGROUPS) { + // --- Load scale pair for this (n2, group) --- + const int group_idx = k / blocks_per_group; + vec2 scale_pair = load_scale_pair(n2, group_idx, N2); + float scale0 = scale_pair.x; + float scale1 = scale_pair.y; + + $if USE_SUBGROUP_BROADCAST: + // --- Load 8 activations per participating lane (only lanes 0..3) --- + // Lanes 0..3 each load 2 vec4s; other lanes receive via subgroupBroadcast + // in the dequant loops below. + vec4 in_vecs[2] = vec4[2](vec4(0.0), vec4(0.0)); + if (lane_id < 4u) { + const int vec4_base = k * 8 + int(lane_id) * 2; + in_vecs[0] = load_input_vec4(vec4_base); + in_vecs[1] = load_input_vec4(vec4_base + 1); + } + $else: + // --- Load all 8 activation vec4s per thread (no subgroup broadcast) --- + // Each thread independently reads the 32 activations (8 vec4) for this + // k-block. All lanes hit the same addresses, so L1 serves ~1 load from + // DRAM per unique address across the wave. + vec4 in_vecs[8]; + const int vec4_base = k * 8; + [[unroll]] for (int i = 0; i < 8; ++i) { + in_vecs[i] = load_input_vec4(vec4_base + i); + } + + // --- Load 4 int32s for the "hi" half (K positions 0..15) --- + int w_pack0 = t_q4_weights[n2 + k * K_BLOCK_STRIDE_W + N2 * 0]; + int w_pack1 = t_q4_weights[n2 + k * K_BLOCK_STRIDE_W + N2 * 1]; + int w_pack2 = t_q4_weights[n2 + k * K_BLOCK_STRIDE_W + N2 * 2]; + int w_pack3 = t_q4_weights[n2 + k * K_BLOCK_STRIDE_W + N2 * 3]; + + // --- Dequant + accumulate for the "hi" block (K = 0..15 of this K_PER_STEP). --- + // Each regA word contains interleaved byte pairs: byte b = (N_even, K=b) + // in the low nibble, (N_odd, K=b) in the high nibble. + float in_val; + + [[unroll]] for (int k4i = 0; k4i < 4; ++k4i) { + $if USE_SUBGROUP_BROADCAST: + in_val = subgroupBroadcast(in_vecs[0][k4i], 0u); + $else: + in_val = in_vecs[0][k4i]; + acc.x += (float(int((uint(w_pack0) >> (8 * k4i)) & 0xFu)) - 8.0) * scale0 * in_val; + acc.y += (float(int((uint(w_pack0) >> (8 * k4i + 4)) & 0xFu)) - 8.0) * scale1 * in_val; + } + + [[unroll]] for (int k4i = 0; k4i < 4; ++k4i) { + $if USE_SUBGROUP_BROADCAST: + in_val = subgroupBroadcast(in_vecs[1][k4i], 0u); + $else: + in_val = in_vecs[1][k4i]; + acc.x += (float(int((uint(w_pack1) >> (8 * k4i)) & 0xFu)) - 8.0) * scale0 * in_val; + acc.y += (float(int((uint(w_pack1) >> (8 * k4i + 4)) & 0xFu)) - 8.0) * scale1 * in_val; + } + + [[unroll]] for (int k4i = 0; k4i < 4; ++k4i) { + $if USE_SUBGROUP_BROADCAST: + in_val = subgroupBroadcast(in_vecs[0][k4i], 1u); + $else: + in_val = in_vecs[2][k4i]; + acc.x += (float(int((uint(w_pack2) >> (8 * k4i)) & 0xFu)) - 8.0) * scale0 * in_val; + acc.y += (float(int((uint(w_pack2) >> (8 * k4i + 4)) & 0xFu)) - 8.0) * scale1 * in_val; + } + + [[unroll]] for (int k4i = 0; k4i < 4; ++k4i) { + $if USE_SUBGROUP_BROADCAST: + in_val = subgroupBroadcast(in_vecs[1][k4i], 1u); + $else: + in_val = in_vecs[3][k4i]; + acc.x += (float(int((uint(w_pack3) >> (8 * k4i)) & 0xFu)) - 8.0) * scale0 * in_val; + acc.y += (float(int((uint(w_pack3) >> (8 * k4i + 4)) & 0xFu)) - 8.0) * scale1 * in_val; + } + + // --- Load 4 int32s for the "lo" half (K positions 16..31). --- + w_pack0 = t_q4_weights[n2 + k * K_BLOCK_STRIDE_W + N2 * 4]; + w_pack1 = t_q4_weights[n2 + k * K_BLOCK_STRIDE_W + N2 * 5]; + w_pack2 = t_q4_weights[n2 + k * K_BLOCK_STRIDE_W + N2 * 6]; + w_pack3 = t_q4_weights[n2 + k * K_BLOCK_STRIDE_W + N2 * 7]; + + // --- Dequant + accumulate for the "lo" block (K = 16..31). --- + [[unroll]] for (int k4i = 0; k4i < 4; ++k4i) { + $if USE_SUBGROUP_BROADCAST: + in_val = subgroupBroadcast(in_vecs[0][k4i], 2u); + $else: + in_val = in_vecs[4][k4i]; + acc.x += (float(int((uint(w_pack0) >> (8 * k4i)) & 0xFu)) - 8.0) * scale0 * in_val; + acc.y += (float(int((uint(w_pack0) >> (8 * k4i + 4)) & 0xFu)) - 8.0) * scale1 * in_val; + } + + [[unroll]] for (int k4i = 0; k4i < 4; ++k4i) { + $if USE_SUBGROUP_BROADCAST: + in_val = subgroupBroadcast(in_vecs[1][k4i], 2u); + $else: + in_val = in_vecs[5][k4i]; + acc.x += (float(int((uint(w_pack1) >> (8 * k4i)) & 0xFu)) - 8.0) * scale0 * in_val; + acc.y += (float(int((uint(w_pack1) >> (8 * k4i + 4)) & 0xFu)) - 8.0) * scale1 * in_val; + } + + [[unroll]] for (int k4i = 0; k4i < 4; ++k4i) { + $if USE_SUBGROUP_BROADCAST: + in_val = subgroupBroadcast(in_vecs[0][k4i], 3u); + $else: + in_val = in_vecs[6][k4i]; + acc.x += (float(int((uint(w_pack2) >> (8 * k4i)) & 0xFu)) - 8.0) * scale0 * in_val; + acc.y += (float(int((uint(w_pack2) >> (8 * k4i + 4)) & 0xFu)) - 8.0) * scale1 * in_val; + } + + [[unroll]] for (int k4i = 0; k4i < 4; ++k4i) { + $if USE_SUBGROUP_BROADCAST: + in_val = subgroupBroadcast(in_vecs[1][k4i], 3u); + $else: + in_val = in_vecs[7][k4i]; + acc.x += (float(int((uint(w_pack3) >> (8 * k4i)) & 0xFu)) - 8.0) * scale0 * in_val; + acc.y += (float(int((uint(w_pack3) >> (8 * k4i + 4)) & 0xFu)) - 8.0) * scale1 * in_val; + } + } + + // --- Inter-wave reduction via flat shared memory (matches OpenCL) --- + if (k_wave_id >= 1) { + partial_sums[(k_wave_id - 1) * LWG_X_SIZE + int(lane_id)] = acc; + } + barrier(); + if (k_wave_id == 0) { + [[unroll]] for (int w = 0; w < NUM_SUBGROUPS - 1; ++w) { + acc += partial_sums[w * LWG_X_SIZE + int(lane_id)]; + } + + // Apply bias if present + if (apply_bias > 0) { + acc.x += float(t_bias[n2 * 2]); + acc.y += float(t_bias[n2 * 2 + 1]); + } + } + + // --- Write 2 outputs --- +#ifdef IO_BUFFER + if (k_wave_id == 0) { + t_output[n2 * 2] = T(acc.x); + t_output[n2 * 2 + 1] = T(acc.y); + } +#else + // texture3d: output stored as width-packed vec4 at (n4, 0, 0). + // Each thread owns 2 outputs (n2*2, n2*2+1). Two consecutive n2s share + // one vec4; only the even-n2 thread assembles and writes the full vec4. + $if USE_SUBGROUP_BROADCAST: + vec2 partner = vec2( + subgroupShuffleXor(acc.x, 1u), + subgroupShuffleXor(acc.y, 1u)); + if (k_wave_id == 0 && (n2 & 1) == 0) { + vec4 out_vec; + out_vec.xy = acc; + out_vec.zw = partner; + const int n4 = n2 / 2; + imageStore(t_output, ivec3(n4, 0, 0), out_vec); + } + $else: + // Subgroup-free partner exchange via shared memory. Only k_wave_id==0 + // threads have a valid reduced `acc`, so only those threads write the + // partner slot; all threads must reach the barrier (uniform control + // flow). Then the even-n2 k_wave_id==0 threads read the n2-XOR-1 + // partner slot and assemble the output vec4. A barrier before the + // write resynchronizes after the inter-wave reduction read of + // partial_sums (which conflicts with the partner-exchange shared + // memory only conceptually — they are separate arrays — but the + // pre-barrier matches the OpenCL reference style and is cheap). + barrier(); + if (k_wave_id == 0) { + nosg_n2_partner[lane_id] = acc; + } + barrier(); + if (k_wave_id == 0 && (n2 & 1) == 0) { + vec2 partner = nosg_n2_partner[lane_id ^ 1u]; + vec4 out_vec; + out_vec.xy = acc; + out_vec.zw = partner; + const int n4 = n2 / 2; + imageStore(t_output, ivec3(n4, 0, 0), out_vec); + } +#endif +} diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/glsl/q4gsw_linear_gemv__w_4x8.yaml b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/glsl/q4gsw_linear_gemv__w_4x8.yaml new file mode 100644 index 00000000000..d5e853d2aed --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q4gsw/glsl/q4gsw_linear_gemv__w_4x8.yaml @@ -0,0 +1,40 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +q4gsw_linear_gemv__w_4x8: + parameter_names_with_default_values: + DTYPE: float + IO_STORAGE: buffer + NUM_SUBGROUPS: 4 + SUBGROUP_SIZE: 64 + LWG_X_SIZE: 64 + USE_SUBGROUP_BROADCAST: true + WEIGHT_TILE_CONTIG_DIM: 0 + generate_variant_forall: + IO_STORAGE: + - VALUE: buffer + - VALUE: texture3d + DTYPE: + - VALUE: float + - VALUE: half + shader_variants: + # interleaved nibble, N dim contiguous, subgroup broadcast. + # SUBGROUP_SIZE=64 (inherited from top-level) auto-pins the pipeline's + # required subgroup size to 64 so subgroupBroadcast indexing matches the + # host-side LWG.x assumption (currently 64). Without this the driver + # may pick any size in [minSubgroupSize, maxSubgroupSize] (e.g. 128 on + # Adreno 750) and break the reduction. + - NAME: q4gsw_linear_gemv__w_4x8_nc + WEIGHT_TILE_CONTIG_DIM: 0 + # interleaved nibble, N dim contiguous, no subgroup ops — portable + # across any subgroup width (works on Mali / 16-wide as well as Adreno). + # SUBGROUP_SIZE=0 opts out of the pipeline subgroup-size pin (the + # SUBGROUP_SIZE macro is unused on the nosg path; shared-mem layout uses + # LWG_X_SIZE). + - NAME: q4gsw_linear_gemv__w_4x8_nc_nosg + WEIGHT_TILE_CONTIG_DIM: 0 + USE_SUBGROUP_BROADCAST: false + SUBGROUP_SIZE: 0 diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q8csw/Q8cswLinearReference.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q8csw/Q8cswLinearReference.cpp new file mode 100644 index 00000000000..e0a9243a883 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q8csw/Q8cswLinearReference.cpp @@ -0,0 +1,83 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include +#include +#include +#include +namespace vkcompute::shader_profiling::reference { +static Tensor q8csw_linear_reference( + const Json& spec, + const TensorMap& tensors, + size_t threads) { + const bool quantized = + spec.at("operator") == "et_vk.linear_q8ta_q8csw.default"; + const size_t wi = quantized ? 3 : 1, si = quantized ? 5 : 2, + bi = quantized ? 6 : 3; + Tensor input = tensor_arg(spec, 0, tensors), + weight = tensor_arg(spec, wi, tensors); + const Tensor& scales = tensor_arg(spec, si, tensors); + const Tensor* bias = spec.at("args").at(bi).is_null() + ? nullptr + : &tensor_arg(spec, bi, tensors); + if (quantized) { + const float scale = q8ta::scale(spec, 1); + const int64_t zp = q8ta::integer(spec, 2); + require( + weight.sizes.size() == 2 && input.sizes.back() == weight.sizes[1], + "Quantized linear weight dimensions must match input features"); + const int64_t k = input.sizes.back(), n = weight.sizes[0]; + const int64_t m = int64_t(input.values.size()) / k; + require( + scales.values.size() == size_t(n) && + (!bias || bias->values.size() == size_t(n)), + "Quantized linear scales and bias must match output features"); + Tensor output{input.sizes, std::vector(size_t(m * n))}; + output.sizes.back() = n; + std::vector quantized_input(input.values.size()); + for (size_t i = 0; i < quantized_input.size(); ++i) + quantized_input[i] = + int16_t(q8ta::quantize(float(input.values[i]), scale, zp) - zp); + std::vector packed_weight(weight.values.size()); + for (int64_t col = 0; col < n; ++col) + for (int64_t j = 0; j < k; ++j) + packed_weight[size_t(j * n + col)] = + int8_t(weight.values[size_t(col * k + j)]); + constexpr int64_t tile = 32; + const int64_t blocks = (n + tile - 1) / tile; + parallel_for(size_t(m * blocks), threads, [&](size_t index) { + const int64_t row = int64_t(index) / blocks; + const int64_t start = int64_t(index) % blocks * tile; + const int64_t count = std::min(tile, n - start); + std::array sums{}; + for (int64_t j = 0; j < k; ++j) { + const int64_t value = quantized_input[size_t(row * k + j)]; + for (int64_t col = 0; col < count; ++col) + sums[size_t(col)] += + value * packed_weight[size_t(j * n + start + col)]; + } + for (int64_t col = 0; col < count; ++col) { + const size_t oc = size_t(start + col); + const float combined_scale = scale * float(scales.values[oc]); + output.values[size_t(row * n) + oc] = + std::fma(float(sums[size_t(col)]), combined_scale, 0.0f) + + (bias ? float(bias->values[oc]) : 0.0f); + } + }); + return output; + } + for (size_t i = 0; i < weight.values.size(); ++i) + weight.values[i] = float(weight.values[i]) * + float(scales.values[i / size_t(weight.sizes[1])]); + return linear_projection(input, weight, bias, threads); +} +REGISTER_REFERENCE_FUNCTIONS { + REGISTER_REFERENCE_FN(q8csw_linear, q8csw_linear_reference); + REGISTER_OPERATOR_REFERENCE(et_vk.linear_q8csw.default, q8csw_linear); + REGISTER_OPERATOR_REFERENCE(et_vk.linear_q8ta_q8csw.default, q8csw_linear); +} +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q8csw/cases/correctness.json b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q8csw/cases/correctness.json new file mode 100644 index 00000000000..49641442f3f --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q8csw/cases/correctness.json @@ -0,0 +1,1382 @@ +[ + { + "name": "q8csw_linear/correctness", + "operator": "et_vk.linear_q8ta_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 64], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4, 32], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_linear/correctness (2)", + "operator": "et_vk.linear_q8ta_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 128], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_linear/correctness (3)", + "operator": "et_vk.linear_q8ta_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4, 256], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_linear/correctness (4)", + "operator": "et_vk.linear_q8ta_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 64], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [32], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [32, 32], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_linear/correctness (5)", + "operator": "et_vk.linear_q8ta_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 128], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [32, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_linear/correctness (6)", + "operator": "et_vk.linear_q8ta_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 256], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [32, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_linear/correctness (7)", + "operator": "et_vk.linear_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 64], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4, 32], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_linear/correctness (8)", + "operator": "et_vk.linear_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 128], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_linear/correctness (9)", + "operator": "et_vk.linear_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4, 256], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_linear/correctness (10)", + "operator": "et_vk.linear_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [32, 64], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [32], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [32, 32], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_linear/correctness (11)", + "operator": "et_vk.linear_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 128], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [32, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_linear/correctness (12)", + "operator": "et_vk.linear_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 256], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [32, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_linear/correctness (13)", + "operator": "et_vk.linear_q8ta_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 128], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [64], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [32, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_linear/correctness (14)", + "operator": "et_vk.linear_q8ta_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 256], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [128], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [32, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_linear/correctness (15)", + "operator": "et_vk.linear_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 128], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [32, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_linear/correctness (16)", + "operator": "et_vk.linear_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 256], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [32, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q8csw/cases/debug.json b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q8csw/cases/debug.json new file mode 100644 index 00000000000..d6f0f530173 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q8csw/cases/debug.json @@ -0,0 +1,100 @@ +[ + { + "name": "q8csw_linear/debug", + "operator": "et_vk.linear_q8ta_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4, 4], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [4, 4], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [4], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [4], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4, 4], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q8csw/cases/performance.json b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q8csw/cases/performance.json new file mode 100644 index 00000000000..14afdb25816 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/linear/qw/q8csw/cases/performance.json @@ -0,0 +1,190 @@ +[ + { + "name": "q8csw_linear/performance", + "operator": "et_vk.linear_q8ta_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "sweep_variables": { + "v_SIZES": [ + [256, 2048], + [512, 2048], + [1024, 2048] + ], + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": "v_SIZES", + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 0.00800000037997961 + }, + { + "int": -2 + }, + { + "tensor": { + "name": "weight", + "sizes": [2048, 2048], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_sums", + "sizes": [2048], + "dtype": "int32", + "storage": "constant", + "layout": "width", + "data": { + "row_sum": "weight" + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [2048], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [2048], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": "v_SIZES", + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8csw_linear/performance (2)", + "operator": "et_vk.linear_q8csw.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0020000000949949026, + "rtol": 0.0010000000474974513 + }, + "sweep_variables": { + "v_SIZES": [ + [256, 2048], + [512, 2048], + [1024, 2048] + ], + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": "v_SIZES", + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [2048, 2048], + "dtype": "int8", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "tensor": { + "name": "weight_scales", + "sizes": [2048], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [0.004999999888241291, 0.014999999664723873] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [2048], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": "v_SIZES", + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/matmul/MatmulReference.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/matmul/MatmulReference.cpp new file mode 100644 index 00000000000..bc3efa771dd --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/matmul/MatmulReference.cpp @@ -0,0 +1,61 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include +#include +namespace vkcompute::shader_profiling::reference { +Tensor +matmul_reference(const Json& spec, const TensorMap& tensors, size_t threads) { + const bool add = spec.at("operator") == "aten.addmm.default"; + const Tensor& a = tensor_arg(spec, add ? 1 : 0, tensors); + const Tensor& b = tensor_arg(spec, add ? 2 : 1, tensors); + require( + (a.sizes.size() == 2 || a.sizes.size() == 3) && + a.sizes.size() == b.sizes.size(), + "Matmul expects two rank2 or rank3 inputs"); + const size_t k = a.sizes.back(), m = a.sizes[a.sizes.size() - 2], + n = b.sizes.back(); + const size_t batches = a.sizes.size() == 3 ? a.sizes[0] : 1; + require( + b.sizes[b.sizes.size() - 2] == int64_t(k) && + (a.sizes.size() == 2 || a.sizes[0] == b.sizes[0]), + "Matmul dimensions differ"); + Tensor out{a.sizes, std::vector(batches * m * n)}; + out.sizes.back() = n; + const Tensor* bias = add ? &tensor_arg(spec, 0, tensors) : nullptr; + const float beta = + add ? spec.at("args").at(3).at("float").get() : 0.0f; + const float alpha = + add ? spec.at("args").at(4).at("float").get() : 1.0f; + require( + !bias || bias->values.size() == n, + "Addmm reference requires vector bias"); + for (size_t batch = 0; batch < batches; ++batch) { + Tensor input{ + {int64_t(m), int64_t(k)}, + std::vector( + a.values.begin() + batch * m * k, + a.values.begin() + (batch + 1) * m * k)}; + Tensor weight{{int64_t(n), int64_t(k)}, std::vector(n * k)}; + parallel_for(n, threads, [&](size_t j) { + for (size_t i = 0; i < k; ++i) + weight.values[j * k + i] = b.values[(batch * k + i) * n + j]; + }); + Tensor result = linear_projection(input, weight, nullptr, threads); + for (size_t i = 0; i < m * n; ++i) + out.values[batch * m * n + i] = alpha * float(result.values[i]) + + (bias ? beta * float(bias->values[i % n]) : 0.0f); + } + return out; +} +REGISTER_REFERENCE_FUNCTIONS { + REGISTER_REFERENCE_FN(matmul, matmul_reference); + REGISTER_OPERATOR_REFERENCE(aten.mm.default, matmul); + REGISTER_OPERATOR_REFERENCE(aten.bmm.default, matmul); + REGISTER_OPERATOR_REFERENCE(aten.addmm.default, matmul); +} +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/matmul/MatmulReference.h b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/matmul/MatmulReference.h new file mode 100644 index 00000000000..4181a0547b9 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/matmul/MatmulReference.h @@ -0,0 +1,16 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include + +namespace vkcompute::shader_profiling::reference { + +Tensor +matmul_reference(const Json& spec, const TensorMap& tensors, size_t threads); + +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/matmul/MatmulRoutes.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/matmul/MatmulRoutes.cpp new file mode 100644 index 00000000000..97cf9d87b2b --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/matmul/MatmulRoutes.cpp @@ -0,0 +1,84 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include +#include +#include +#include +#include +#include +namespace vkcompute::shader_profiling { +namespace { +template +Json build_gemm_route( + ComputeGraph& graph, + const std::vector& args, + const Json& options) { + if (!options.empty()) + throw std::invalid_argument("GEMM routes accept no options"); + const ValueRef input = args.at(0), weight = args.at(1), out = args.back(); + if constexpr (Coop) { + const auto* adapter = graph.context()->adapter_ptr(); + const auto sizes = graph.sizes_of(input), outs = graph.sizes_of(out); + if (!adapter->supports_cooperative_matrix() || + adapter->subgroup_size() != 64 || !graph.is_buffer_storage(out) || + sizes.size() != 2 || sizes[0] % 64 || sizes[1] % 32 || outs.back() % 64) + throw UnsupportedRoute( + "coopmat requires cooperative matrices, subgroup64, buffer output, and M/N/K alignment64/64/32"); + } + if (Linear || graph.val_is_tref(weight)) { + const int64_t batch = + graph.sizes_of(weight).size() == 3 ? graph.sizes_of(weight).at(0) : 1; + const ValueRef packed = + prepack_fp_linear_weight(graph, weight, Linear, batch, Coop); + const bool has_bias = Linear && graph.val_is_not_none(args.at(2)); + ValueRef bias = kDummyValueRef; + if (has_bias) + bias = prepack_standard( + graph, + args.at(2), + graph.storage_type_of(out), + utils::kWidthPacked, + false); + if constexpr (Coop) + add_linear_coopmat_node( + graph, + input, + packed, + bias, + has_bias, + out, + utils::safe_downcast(batch)); + else + add_linear_tiled_node( + graph, + input, + packed, + bias, + has_bias, + out, + utils::safe_downcast(batch)); + } else { + if constexpr (Coop) + add_matmul_coopmat_node(graph, input, weight, out); + else + add_matmul_tiled_node(graph, input, weight, out); + } + return {{"name", Coop ? "coopmat" : "tiled"}, {"options", options}}; +} +REGISTER_OPERATOR_ROUTES { + REGISTER_OPERATOR_ROUTE( + aten.mm.default, tiled, (build_gemm_route)); + REGISTER_OPERATOR_ROUTE( + aten.mm.default, coopmat, (build_gemm_route)); + REGISTER_OPERATOR_ROUTE( + aten.linear.default, tiled, (build_gemm_route)); + REGISTER_OPERATOR_ROUTE( + aten.linear.default, coopmat, (build_gemm_route)); +} +} // namespace +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/matmul/MatmulThroughput.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/matmul/MatmulThroughput.cpp new file mode 100644 index 00000000000..2f69c7c7db1 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/matmul/MatmulThroughput.cpp @@ -0,0 +1,32 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +namespace vkcompute::shader_profiling { +namespace { +double matmul_flops(const Json& spec) { + const size_t input_index = + spec.at("operator") == "aten.addmm.default" ? 1 : 0; + const auto sizes = throughput::tensor_argument(spec, input_index) + .at("sizes") + .get>(); + return 2.0 * sizes.back() * + throughput::tensor_elements(throughput::output_tensor(spec)); +} +REGISTER_THROUGHPUT_FUNCTIONS { + REGISTER_OPERATOR_MEMORY_TRAFFIC(aten.mm.default, throughput::tensor_traffic); + REGISTER_OPERATOR_COMPUTE_FLOPS(aten.mm.default, matmul_flops); + REGISTER_OPERATOR_MEMORY_TRAFFIC( + aten.bmm.default, throughput::tensor_traffic); + REGISTER_OPERATOR_COMPUTE_FLOPS(aten.bmm.default, matmul_flops); + REGISTER_OPERATOR_MEMORY_TRAFFIC( + aten.addmm.default, throughput::tensor_traffic); + REGISTER_OPERATOR_COMPUTE_FLOPS(aten.addmm.default, matmul_flops); +} + +} // namespace +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/matmul/cases/correctness.json b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/matmul/cases/correctness.json new file mode 100644 index 00000000000..f5286476345 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/matmul/cases/correctness.json @@ -0,0 +1,3681 @@ +[ + { + "name": "mm correctness [0,64,128,64] aten.mm.default texture3d float32", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 64], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [64, 64], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,64,128,47] aten.mm.default texture3d float32", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 47], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [64, 47], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,64,128,64] aten.mm.default buffer float32 float32 buffer width", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "tiled"], + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 64], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [64, 64], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,64,128,47] aten.mm.default buffer float32", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 47], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [64, 47], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,128,256,128] aten.mm.default texture3d float32", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [128, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 128], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [128, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,128,256,128] aten.mm.default buffer float32 float32 buffer width", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "tiled"], + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [128, 256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 128], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [128, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,32,64,256] aten.mm.default texture3d float32", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 64], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 256], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [32, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,32,64,256] aten.mm.default buffer float32", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 64], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 256], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [32, 256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,57,131,43] aten.mm.default texture3d float32", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [57, 131], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [131, 43], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [57, 43], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,57,131,43] aten.mm.default buffer float32", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [57, 131], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [131, 43], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [57, 43], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,33,67,91] aten.mm.default texture3d float32", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [33, 67], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [67, 91], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [33, 91], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,33,67,91] aten.mm.default buffer float32", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [33, 67], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [67, 91], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [33, 91], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,64,47,128] aten.mm.default texture3d float32", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 47], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [47, 128], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [64, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,64,47,128] aten.mm.default buffer float32", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 47], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [47, 128], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [64, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,64,128,64] aten.mm.default buffer float32 float32 buffer width coopmat", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "coopmat", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 64], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [64, 64], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,128,256,128] aten.mm.default buffer float32 float32 buffer width coopmat", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "coopmat", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [128, 256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 128], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [128, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [1,64,128,64] aten.bmm.default texture3d float32", + "operator": "aten.bmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [1, 128, 64], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 64], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [1,64,128,64] aten.bmm.default buffer float32", + "operator": "aten.bmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 64, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [1, 128, 64], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [1, 64, 64], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [1,4,32,16] aten.bmm.default texture3d float32", + "operator": "aten.bmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 4, 32], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [1, 32, 16], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [1, 4, 16], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [1,4,32,16] aten.bmm.default buffer float32", + "operator": "aten.bmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 4, 32], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [1, 32, 16], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [1, 4, 16], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [1,19,53,37] aten.bmm.default texture3d float32", + "operator": "aten.bmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 19, 53], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [1, 53, 37], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [1, 19, 37], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [1,19,53,37] aten.bmm.default buffer float32", + "operator": "aten.bmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 19, 53], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [1, 53, 37], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [1, 19, 37], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,64,128,64] aten.addmm.default texture3d float32", + "operator": "aten.addmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "input", + "sizes": [64, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 64], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 1.0 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [64, 64], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,64,128,64] aten.addmm.default buffer float32", + "operator": "aten.addmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "input", + "sizes": [64, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 64], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 1.0 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [64, 64], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,128,256,128] aten.addmm.default texture3d float32", + "operator": "aten.addmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "input", + "sizes": [128, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 128], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 1.0 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [128, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,64,47,128] aten.addmm.default texture3d float32", + "operator": "aten.addmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "input", + "sizes": [64, 47], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [47, 128], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 1.0 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [64, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,128,256,128] aten.addmm.default buffer float32", + "operator": "aten.addmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "input", + "sizes": [128, 256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 128], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 1.0 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [128, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,64,47,128] aten.addmm.default buffer float32", + "operator": "aten.addmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "input", + "sizes": [64, 47], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [47, 128], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 1.0 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [64, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,32,64,256] aten.addmm.default texture3d float32", + "operator": "aten.addmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "bias", + "sizes": [256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "input", + "sizes": [32, 64], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 256], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 1.0 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [32, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,32,64,256] aten.addmm.default buffer float32", + "operator": "aten.addmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "bias", + "sizes": [256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "input", + "sizes": [32, 64], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 256], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 1.0 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [32, 256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,57,131,43] aten.addmm.default texture3d float32", + "operator": "aten.addmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "bias", + "sizes": [43], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "input", + "sizes": [57, 131], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [131, 43], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 1.0 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [57, 43], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,57,131,43] aten.addmm.default buffer float32", + "operator": "aten.addmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "bias", + "sizes": [43], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "input", + "sizes": [57, 131], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [131, 43], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 1.0 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [57, 43], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,33,67,91] aten.addmm.default texture3d float32", + "operator": "aten.addmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "bias", + "sizes": [91], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "input", + "sizes": [33, 67], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [67, 91], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 1.0 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [33, 91], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,33,67,91] aten.addmm.default buffer float32", + "operator": "aten.addmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "bias", + "sizes": [91], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "input", + "sizes": [33, 67], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [67, 91], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 1.0 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [33, 91], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,64,128,47] aten.addmm.default texture3d float32", + "operator": "aten.addmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "bias", + "sizes": [47], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "input", + "sizes": [64, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 47], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 1.0 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [64, 47], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,64,128,47] aten.addmm.default buffer float32", + "operator": "aten.addmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "bias", + "sizes": [47], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "input", + "sizes": [64, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 47], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 1.0 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [64, 47], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,64,128,64] aten.linear.default", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [64, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,64,128,47] aten.linear.default", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [47, 128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [64, 47], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,128,256,128] aten.linear.default", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [128, 256], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [128, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,32,64,256] aten.linear.default", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [32, 256], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,57,131,43] aten.linear.default", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [57, 131], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [43, 131], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [57, 43], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,33,67,91] aten.linear.default", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [33, 67], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [91, 67], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [33, 91], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,64,47,128] aten.linear.default", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 47], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 47], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [64, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,64,128,64] aten.linear.default buffer float32 constant=False tiled", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "tiled", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [64, 64], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,128,256,128] aten.linear.default buffer float32 constant=False tiled", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "tiled", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [128, 256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [128, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,64,128,64] aten.linear.default buffer float32 constant=False coopmat", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "coopmat", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [64, 64], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,128,256,128] aten.linear.default buffer float32 constant=False coopmat", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "coopmat", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [128, 256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [128, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,64,128,64] aten.linear.default (4)", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [64, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,64,128,47] aten.linear.default (2)", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [47, 128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [47], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [64, 47], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,128,256,128] aten.linear.default (4)", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [128, 256], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [128, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,32,64,256] aten.linear.default (2)", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [256], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [32, 256], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,57,131,43] aten.linear.default (2)", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [57, 131], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [43, 131], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [43], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [57, 43], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,33,67,91] aten.linear.default (2)", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [33, 67], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [91, 67], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [91], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [33, 91], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,64,47,128] aten.linear.default (2)", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 47], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 47], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [64, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,64,128,64] aten.linear.default buffer float32 constant=False tiled (2)", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "tiled", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [64, 64], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,128,256,128] aten.linear.default buffer float32 constant=False tiled (2)", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "tiled", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [128, 256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [128, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,64,128,64] aten.linear.default buffer float32 constant=False coopmat (2)", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "coopmat", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [64, 64], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm correctness [0,128,256,128] aten.linear.default buffer float32 constant=False coopmat (2)", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "coopmat", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [128, 256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [128, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/gemm/matmul/cases/performance.json b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/matmul/cases/performance.json new file mode 100644 index 00000000000..70cf7953aa0 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/gemm/matmul/cases/performance.json @@ -0,0 +1,5502 @@ +[ + { + "name": "mm performance [0,4096,1024,256] aten.mm.default texture3d float32", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 1024], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [1024, 256], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,1024,256] aten.mm.default buffer float32 float32 buffer width", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "tiled"], + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 1024], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [1024, 256], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,256,128] aten.mm.default texture3d float32", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 128], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,256,128] aten.mm.default buffer float32 float32 buffer width", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "tiled"], + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 128], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,128,256] aten.mm.default texture3d float32", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,128,256] aten.mm.default buffer float32 float32 buffer width", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "tiled"], + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,64,128] aten.mm.default texture3d float32", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 64], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 128], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,64,128] aten.mm.default buffer float32 float32 buffer width", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "tiled"], + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 64], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 128], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,1024,256] aten.mm.default buffer float32 float32 buffer width coopmat", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "coopmat", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 1024], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [1024, 256], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,256,128] aten.mm.default buffer float32 float32 buffer width coopmat", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "coopmat", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 128], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,128,256] aten.mm.default buffer float32 float32 buffer width coopmat", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "coopmat", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,64,128] aten.mm.default buffer float32 float32 buffer width coopmat", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "coopmat", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 64], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 128], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,1024,256] aten.mm.default texture3d float16", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 1024], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [1024, 256], + "dtype": "float16", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,1024,256] aten.mm.default buffer float16", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "tiled", "coopmat"], + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 1024], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [1024, 256], + "dtype": "float16", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,256,128] aten.mm.default texture3d float16", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 256], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 128], + "dtype": "float16", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,256,128] aten.mm.default buffer float16", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "tiled", "coopmat"], + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 256], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 128], + "dtype": "float16", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,128,256] aten.mm.default texture3d float16", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "float16", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,128,256] aten.mm.default buffer float16", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "tiled", "coopmat"], + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 128], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "float16", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,64,128] aten.mm.default texture3d float16", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 64], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 128], + "dtype": "float16", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,64,128] aten.mm.default buffer float16", + "operator": "aten.mm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "tiled", "coopmat"], + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 64], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 128], + "dtype": "float16", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [1,4096,256,1024] aten.bmm.default texture3d float32", + "operator": "aten.bmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 4096, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [1, 256, 1024], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [1, 4096, 1024], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [1,4096,256,128] aten.bmm.default texture3d float32", + "operator": "aten.bmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 4096, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [1, 256, 128], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [1, 4096, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [1,4096,256,1024] aten.bmm.default buffer float32", + "operator": "aten.bmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 4096, 256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [1, 256, 1024], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [1, 4096, 1024], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [1,4096,256,128] aten.bmm.default buffer float32", + "operator": "aten.bmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 4096, 256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [1, 256, 128], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [1, 4096, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [1,256,4096,64] aten.bmm.default texture3d float32", + "operator": "aten.bmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 256, 4096], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [1, 4096, 64], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [1, 256, 64], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [1,256,4096,64] aten.bmm.default buffer float32", + "operator": "aten.bmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 256, 4096], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [1, 4096, 64], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [1, 256, 64], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [1,4096,256,1024] aten.bmm.default texture3d float16", + "operator": "aten.bmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 4096, 256], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [1, 256, 1024], + "dtype": "float16", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [1, 4096, 1024], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [1,4096,256,128] aten.bmm.default texture3d float16", + "operator": "aten.bmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 4096, 256], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [1, 256, 128], + "dtype": "float16", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [1, 4096, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [1,4096,256,1024] aten.bmm.default buffer float16", + "operator": "aten.bmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 4096, 256], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [1, 256, 1024], + "dtype": "float16", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [1, 4096, 1024], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [1,4096,256,128] aten.bmm.default buffer float16", + "operator": "aten.bmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 4096, 256], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [1, 256, 128], + "dtype": "float16", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [1, 4096, 128], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [1,256,4096,64] aten.bmm.default texture3d float16", + "operator": "aten.bmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 256, 4096], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [1, 4096, 64], + "dtype": "float16", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [1, 256, 64], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [1,256,4096,64] aten.bmm.default buffer float16", + "operator": "aten.bmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 256, 4096], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [1, 4096, 64], + "dtype": "float16", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [1, 256, 64], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,1024,256] aten.addmm.default texture3d float32", + "operator": "aten.addmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "bias", + "sizes": [256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "input", + "sizes": [4096, 1024], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [1024, 256], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 1.0 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,128,256] aten.addmm.default texture3d float32", + "operator": "aten.addmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "bias", + "sizes": [256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "input", + "sizes": [4096, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 1.0 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,1024,256] aten.addmm.default buffer float32", + "operator": "aten.addmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "bias", + "sizes": [256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "input", + "sizes": [4096, 1024], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [1024, 256], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 1.0 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,128,256] aten.addmm.default buffer float32", + "operator": "aten.addmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "bias", + "sizes": [256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "input", + "sizes": [4096, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 1.0 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,256,128] aten.addmm.default texture3d float32", + "operator": "aten.addmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "input", + "sizes": [4096, 256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 128], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 1.0 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,64,128] aten.addmm.default texture3d float32", + "operator": "aten.addmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "input", + "sizes": [4096, 64], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 128], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 1.0 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,256,128] aten.addmm.default buffer float32", + "operator": "aten.addmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "input", + "sizes": [4096, 256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 128], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 1.0 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,64,128] aten.addmm.default buffer float32", + "operator": "aten.addmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "input", + "sizes": [4096, 64], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 128], + "dtype": "float32", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 1.0 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,1024,256] aten.addmm.default texture3d float16", + "operator": "aten.addmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "bias", + "sizes": [256], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "input", + "sizes": [4096, 1024], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [1024, 256], + "dtype": "float16", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 1.0 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,128,256] aten.addmm.default texture3d float16", + "operator": "aten.addmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "bias", + "sizes": [256], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "input", + "sizes": [4096, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "float16", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 1.0 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,1024,256] aten.addmm.default buffer float16", + "operator": "aten.addmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "bias", + "sizes": [256], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "input", + "sizes": [4096, 1024], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [1024, 256], + "dtype": "float16", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 1.0 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,128,256] aten.addmm.default buffer float16", + "operator": "aten.addmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "bias", + "sizes": [256], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "input", + "sizes": [4096, 128], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "float16", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 1.0 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,256,128] aten.addmm.default texture3d float16", + "operator": "aten.addmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "input", + "sizes": [4096, 256], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 128], + "dtype": "float16", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 1.0 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,64,128] aten.addmm.default texture3d float16", + "operator": "aten.addmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["texture3d", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "input", + "sizes": [4096, 64], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 128], + "dtype": "float16", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 1.0 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,256,128] aten.addmm.default buffer float16", + "operator": "aten.addmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "input", + "sizes": [4096, 256], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 128], + "dtype": "float16", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 1.0 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,64,128] aten.addmm.default buffer float16", + "operator": "aten.addmm.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "auto", + "sweep_variables": { + "v_WEIGHT_STORAGE": ["buffer", "constant"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "input", + "sizes": [4096, 64], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [64, 128], + "dtype": "float16", + "storage": "v_WEIGHT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 1.0 + }, + { + "float": 1.0 + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,1024,256] aten.linear.default", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 1024], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 1024], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,256,128] aten.linear.default", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 256], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,128,256] aten.linear.default", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,64,128] aten.linear.default", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,1024,256] aten.linear.default buffer float32 constant=False tiled", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "tiled", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 1024], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 1024], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,256,128] aten.linear.default buffer float32 constant=False tiled", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "tiled", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,128,256] aten.linear.default buffer float32 constant=False tiled", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "tiled", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,64,128] aten.linear.default buffer float32 constant=False tiled", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "tiled", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 64], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,1024,256] aten.linear.default buffer float32 constant=False coopmat", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "coopmat", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 1024], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 1024], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,256,128] aten.linear.default buffer float32 constant=False coopmat", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "coopmat", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,128,256] aten.linear.default buffer float32 constant=False coopmat", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "coopmat", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,64,128] aten.linear.default buffer float32 constant=False coopmat", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "coopmat", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 64], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,1024,256] aten.linear.default texture3d float16 constant=False auto", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 1024], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 1024], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,1024,256] aten.linear.default buffer float16 constant=False", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "tiled", "coopmat"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 1024], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 1024], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,256,128] aten.linear.default texture3d float16 constant=False auto", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 256], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,256,128] aten.linear.default buffer float16 constant=False", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "tiled", "coopmat"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 256], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,128,256] aten.linear.default texture3d float16 constant=False auto", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 128], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,128,256] aten.linear.default buffer float16 constant=False", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "tiled", "coopmat"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 128], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 128], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,64,128] aten.linear.default texture3d float16 constant=False auto", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 64], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 64], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,64,128] aten.linear.default buffer float16 constant=False", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "tiled", "coopmat"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 64], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 64], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + null, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,1024,256] aten.linear.default (6)", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 1024], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 1024], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [256], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,256,128] aten.linear.default (6)", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 256], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,128,256] aten.linear.default (6)", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [256], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,64,128] aten.linear.default (6)", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "auto", + "sweep_variables": { + "v_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 64], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float32", + "storage": "v_STORAGE", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,1024,256] aten.linear.default buffer float32 constant=False tiled (2)", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "tiled", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 1024], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 1024], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [256], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,256,128] aten.linear.default buffer float32 constant=False tiled (2)", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "tiled", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,128,256] aten.linear.default buffer float32 constant=False tiled (2)", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "tiled", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [256], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,64,128] aten.linear.default buffer float32 constant=False tiled (2)", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.001, + "rtol": 0.001 + }, + "route": "tiled", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 64], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,1024,256] aten.linear.default buffer float32 constant=False coopmat (2)", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "coopmat", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 1024], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 1024], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [256], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,256,128] aten.linear.default buffer float32 constant=False coopmat (2)", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "coopmat", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,128,256] aten.linear.default buffer float32 constant=False coopmat (2)", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "coopmat", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [256], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,64,128] aten.linear.default buffer float32 constant=False coopmat (2)", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.1, + "rtol": 0.01 + }, + "route": "coopmat", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 64], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 64], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float32", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float32", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,1024,256] aten.linear.default texture3d float16 constant=False auto (2)", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 1024], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 1024], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [256], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,1024,256] aten.linear.default buffer float16 constant=False (2)", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "tiled", "coopmat"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 1024], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 1024], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [256], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,256,128] aten.linear.default texture3d float16 constant=False auto (2)", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 256], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,256,128] aten.linear.default buffer float16 constant=False (2)", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "tiled", "coopmat"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 256], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 256], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,128,256] aten.linear.default texture3d float16 constant=False auto (2)", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 128], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [256], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,128,256] aten.linear.default buffer float16 constant=False (2)", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "tiled", "coopmat"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 128], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [256, 128], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [256], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 256], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,64,128] aten.linear.default texture3d float16 constant=False auto (2)", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "auto", + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 64], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 64], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float16", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "mm performance [0,4096,64,128] aten.linear.default buffer float16 constant=False (2)", + "operator": "aten.linear.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0, + "rtol": 0.1 + }, + "route": "v_ROUTE", + "sweep_variables": { + "v_ROUTE": ["auto", "tiled", "coopmat"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4096, 64], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "weight", + "sizes": [128, 64], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "bias", + "sizes": [128], + "dtype": "float16", + "storage": "constant", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "tensor": { + "name": "output", + "sizes": [4096, 128], + "dtype": "float16", + "storage": "buffer", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/pixel_shuffle/q8ta/Q8taPixelShuffleReference.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/pixel_shuffle/q8ta/Q8taPixelShuffleReference.cpp new file mode 100644 index 00000000000..3c74741ca83 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/pixel_shuffle/q8ta/Q8taPixelShuffleReference.cpp @@ -0,0 +1,49 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include + +namespace vkcompute::shader_profiling::reference { +static Tensor q8ta_pixel_shuffle_reference( + const Json& spec, + const TensorMap& tensors, + size_t threads) { + const Tensor& input = tensor_arg(spec, 0, tensors); + Tensor out = q8ta::output_tensor(spec, spec.at("args").size() - 1); + const float is = q8ta::scale(spec, 1), + inverse_out_scale = q8ta::scale(spec, 3); + const int64_t iz = q8ta::integer(spec, 2), oz = q8ta::integer(spec, 4), + r = q8ta::integer(spec, 5); + const float actual_out_scale = 1.0f / inverse_out_scale; + const bool passthrough = std::abs(is - actual_out_scale) <= + 1e-7f * std::max(std::abs(is), 1e-7f) && + iz == oz; + const int64_t channels = out.sizes[1], height = out.sizes[2], + width = out.sizes[3]; + parallel_for(out.values.size(), threads, [&](size_t i) { + const int64_t x = int64_t(i) % width, y = int64_t(i) / width % height; + const int64_t c = int64_t(i) / (width * height) % channels, + n = int64_t(i) / (width * height * channels); + const int64_t source_c = c * r * r + y % r * r + x % r; + const int64_t source = + ((n * input.sizes[1] + source_c) * input.sizes[2] + y / r) * + input.sizes[3] + + x / r; + const int64_t value = int64_t(input.values[source]); + out.values[i] = passthrough + ? double(value) + : double(q8ta::requantize( + q8ta::dequantize(value, is, iz), inverse_out_scale, oz)); + }); + return out; +} +REGISTER_REFERENCE_FUNCTIONS { + REGISTER_REFERENCE_FN(q8ta_pixel_shuffle, q8ta_pixel_shuffle_reference); + REGISTER_OPERATOR_REFERENCE( + et_vk.q8ta_pixel_shuffle.default, q8ta_pixel_shuffle); +} +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/pixel_shuffle/q8ta/Q8taPixelShuffleThroughput.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/pixel_shuffle/q8ta/Q8taPixelShuffleThroughput.cpp new file mode 100644 index 00000000000..4334d243439 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/pixel_shuffle/q8ta/Q8taPixelShuffleThroughput.cpp @@ -0,0 +1,30 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include + +namespace vkcompute::shader_profiling { +namespace { +double pixel_shuffle_flops(const Json& spec) { + const float in_scale = spec.at("args").at(1).at("float").get(); + const float out_scale = 1.0f / spec.at("args").at(3).at("float").get(); + const bool passthrough = std::abs(in_scale - out_scale) <= + 1e-7f * std::max(std::abs(in_scale), 1e-7f) && + spec.at("args").at(2).at("int") == spec.at("args").at(4).at("int"); + return (passthrough ? 0 : 3) * + throughput::tensor_elements(throughput::output_tensor(spec)); +} +REGISTER_THROUGHPUT_FUNCTIONS { + REGISTER_OPERATOR_MEMORY_TRAFFIC( + et_vk.q8ta_pixel_shuffle.default, throughput::tensor_traffic); + REGISTER_OPERATOR_COMPUTE_FLOPS( + et_vk.q8ta_pixel_shuffle.default, pixel_shuffle_flops); +} +} // namespace +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/pixel_shuffle/q8ta/cases/correctness.json b/backends/vulkan/test/ops/benchmark_v2/operators/pixel_shuffle/q8ta/cases/correctness.json new file mode 100644 index 00000000000..124af8ff463 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/pixel_shuffle/q8ta/cases/correctness.json @@ -0,0 +1,482 @@ +[ + { + "name": "q8ta/pixel_shuffle/correctness", + "operator": "et_vk.q8ta_pixel_shuffle.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.014060718249784, + "rtol": 0 + }, + "sweep_variables": { + "v_INPUT_LAYOUT": ["4w4c", "4c1w"], + "v_OUTPUT_LAYOUT": ["4w4c", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 4, 4], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_INPUT_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00711200013756752 + }, + { + "int": 0 + }, + { + "float": 140.607421875 + }, + { + "int": 0 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 4, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_OUTPUT_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/pixel_shuffle/correctness (2)", + "operator": "et_vk.q8ta_pixel_shuffle.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.014060718249784, + "rtol": 0 + }, + "sweep_variables": { + "v_INPUT_LAYOUT": ["4w4c", "4c1w"], + "v_OUTPUT_LAYOUT": ["4w4c", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 24, 8, 4], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_INPUT_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00711200013756752 + }, + { + "int": 0 + }, + { + "float": 140.607421875 + }, + { + "int": 0 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 6, 16, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_OUTPUT_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/pixel_shuffle/correctness (3)", + "operator": "et_vk.q8ta_pixel_shuffle.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.014060718249784, + "rtol": 0 + }, + "sweep_variables": { + "v_INPUT_LAYOUT": ["4w4c", "4c1w"], + "v_OUTPUT_LAYOUT": ["4w4c", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 12, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_INPUT_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00711200013756752 + }, + { + "int": 0 + }, + { + "float": 140.607421875 + }, + { + "int": 0 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 8, 24, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_OUTPUT_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/pixel_shuffle/correctness (4)", + "operator": "et_vk.q8ta_pixel_shuffle.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.014060718249784, + "rtol": 0 + }, + "sweep_variables": { + "v_INPUT_LAYOUT": ["4w4c", "4c1w"], + "v_OUTPUT_LAYOUT": ["4w4c", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 96, 16, 9], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_INPUT_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00711200013756752 + }, + { + "int": 0 + }, + { + "float": 140.607421875 + }, + { + "int": 0 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 24, 32, 18], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_OUTPUT_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/pixel_shuffle/correctness (5)", + "operator": "et_vk.q8ta_pixel_shuffle.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0076922944664253, + "rtol": 0 + }, + "sweep_variables": { + "v_INPUT_LAYOUT": ["4w4c", "4c1w"], + "v_OUTPUT_LAYOUT": ["4w4c", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 4, 4], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_INPUT_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00711200013756752 + }, + { + "int": 0 + }, + { + "float": 76.9230728149414 + }, + { + "int": 5 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 4, 8, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_OUTPUT_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/pixel_shuffle/correctness (6)", + "operator": "et_vk.q8ta_pixel_shuffle.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0076922944664253, + "rtol": 0 + }, + "sweep_variables": { + "v_INPUT_LAYOUT": ["4w4c", "4c1w"], + "v_OUTPUT_LAYOUT": ["4w4c", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 24, 8, 4], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_INPUT_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00711200013756752 + }, + { + "int": 0 + }, + { + "float": 76.9230728149414 + }, + { + "int": 5 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 6, 16, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_OUTPUT_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/pixel_shuffle/correctness (7)", + "operator": "et_vk.q8ta_pixel_shuffle.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0076922944664253, + "rtol": 0 + }, + "sweep_variables": { + "v_INPUT_LAYOUT": ["4w4c", "4c1w"], + "v_OUTPUT_LAYOUT": ["4w4c", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 32, 12, 8], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_INPUT_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00711200013756752 + }, + { + "int": 0 + }, + { + "float": 76.9230728149414 + }, + { + "int": 5 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 8, 24, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_OUTPUT_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/pixel_shuffle/correctness (8)", + "operator": "et_vk.q8ta_pixel_shuffle.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.0076922944664253, + "rtol": 0 + }, + "sweep_variables": { + "v_INPUT_LAYOUT": ["4w4c", "4c1w"], + "v_OUTPUT_LAYOUT": ["4w4c", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 96, 16, 9], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_INPUT_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00711200013756752 + }, + { + "int": 0 + }, + { + "float": 76.9230728149414 + }, + { + "int": 5 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 24, 32, 18], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_OUTPUT_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/pixel_shuffle/q8ta/cases/performance.json b/backends/vulkan/test/ops/benchmark_v2/operators/pixel_shuffle/q8ta/cases/performance.json new file mode 100644 index 00000000000..e33e0a9327b --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/pixel_shuffle/q8ta/cases/performance.json @@ -0,0 +1,242 @@ +[ + { + "name": "q8ta/pixel_shuffle/performance", + "operator": "et_vk.q8ta_pixel_shuffle.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.014060718249784, + "rtol": 0 + }, + "sweep_variables": { + "v_INPUT_LAYOUT": ["4w4c", "4c1w"], + "v_OUTPUT_LAYOUT": ["4w4c", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 96, 16, 9], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_INPUT_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00711200013756752 + }, + { + "int": 0 + }, + { + "float": 140.607421875 + }, + { + "int": 0 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 24, 32, 18], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_OUTPUT_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/pixel_shuffle/performance (2)", + "operator": "et_vk.q8ta_pixel_shuffle.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.014060718249784, + "rtol": 0 + }, + "sweep_variables": { + "v_INPUT_LAYOUT": ["4w4c", "4c1w"], + "v_OUTPUT_LAYOUT": ["4w4c", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 96, 32, 18], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_INPUT_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00711200013756752 + }, + { + "int": 0 + }, + { + "float": 140.607421875 + }, + { + "int": 0 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 24, 64, 36], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_OUTPUT_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/pixel_shuffle/performance (3)", + "operator": "et_vk.q8ta_pixel_shuffle.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.014060718249784, + "rtol": 0 + }, + "sweep_variables": { + "v_INPUT_LAYOUT": ["4w4c", "4c1w"], + "v_OUTPUT_LAYOUT": ["4w4c", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 96, 64, 36], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_INPUT_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00711200013756752 + }, + { + "int": 0 + }, + { + "float": 140.607421875 + }, + { + "int": 0 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 24, 128, 72], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_OUTPUT_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/pixel_shuffle/performance (4)", + "operator": "et_vk.q8ta_pixel_shuffle.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.014060718249784, + "rtol": 0 + }, + "sweep_variables": { + "v_INPUT_LAYOUT": ["4w4c", "4c1w"], + "v_OUTPUT_LAYOUT": ["4w4c", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 96, 128, 72], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_INPUT_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00711200013756752 + }, + { + "int": 0 + }, + { + "float": 140.607421875 + }, + { + "int": 0 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 24, 256, 144], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_OUTPUT_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/quantization/choose_qparams_per_row/ChooseQparamsReference.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/quantization/choose_qparams_per_row/ChooseQparamsReference.cpp new file mode 100644 index 00000000000..5ebb89e22ca --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/quantization/choose_qparams_per_row/ChooseQparamsReference.cpp @@ -0,0 +1,67 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include +#include +#include +#include +namespace vkcompute::shader_profiling::reference { +ReferenceOutputs choose_qparams_reference( + const Json& spec, + const TensorMap& tensors, + size_t threads) { + const Tensor& input = tensor_arg(spec, 0, tensors); + require(input.sizes.size() == 2, "Choose qparams expects [rows,columns]"); + const int32_t qmin = spec.at("args").at(1).at("int"), + qmax = spec.at("args").at(2).at("int"); + require(qmin < qmax, "Quantization bounds must be increasing"); + const size_t rows = input.sizes[0], columns = input.sizes[1]; + std::vector scales(rows), zeros(rows); + parallel_for(rows, threads, [&](size_t row) { + float lo = 0, hi = 0; + for (size_t j = 0; j < columns; ++j) { + const float value = input.values[row * columns + j]; + lo = std::min(lo, value); + hi = std::max(hi, value); + } + double scale = (double(hi) - lo) / (qmax - qmin); + if (float(scale) == 0 || std::isinf(1.0f / float(scale))) + scale = 0.1; + constexpr float threshold = 6.1e-5f; + if (scale < threshold) { + const float original = scale; + scale = threshold; + if (lo == 0) + hi = threshold * (qmax - qmin); + else if (hi == 0) + lo = -threshold * (qmax - qmin); + else { + lo *= threshold / original; + hi *= threshold / original; + } + } + const double from_lo = qmin - lo / scale, from_hi = qmax - hi / scale; + const double error_lo = std::abs(qmin) - std::abs(lo / scale), + error_hi = std::abs(qmax) - std::abs(hi / scale); + const double initial = error_lo < error_hi ? from_lo : from_hi; + scales[row] = float(scale); + zeros[row] = std::clamp( + double(std::nearbyint(float(initial))), double(qmin), double(qmax)); + }); + return { + {spec.at("args").at(3).at("tensor").get(), + std::move(scales)}, + {spec.at("args").at(4).at("tensor").get(), + std::move(zeros)}}; +} +REGISTER_REFERENCE_FUNCTIONS { + REGISTER_REFERENCE_OUTPUTS_FN( + choose_qparams_per_row, choose_qparams_reference); + REGISTER_OPERATOR_REFERENCE( + etvk.choose_qparams_per_row.default, choose_qparams_per_row); +} +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/quantization/choose_qparams_per_row/ChooseQparamsReference.h b/backends/vulkan/test/ops/benchmark_v2/operators/quantization/choose_qparams_per_row/ChooseQparamsReference.h new file mode 100644 index 00000000000..d2395522788 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/quantization/choose_qparams_per_row/ChooseQparamsReference.h @@ -0,0 +1,18 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include + +namespace vkcompute::shader_profiling::reference { + +ReferenceOutputs choose_qparams_reference( + const Json& spec, + const TensorMap& tensors, + size_t threads); + +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/quantization/choose_qparams_per_row/ChooseQparamsThroughput.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/quantization/choose_qparams_per_row/ChooseQparamsThroughput.cpp new file mode 100644 index 00000000000..44122a260fe --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/quantization/choose_qparams_per_row/ChooseQparamsThroughput.cpp @@ -0,0 +1,25 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +namespace vkcompute::shader_profiling { +namespace { +double choose_qparams_flops(const Json& spec) { + const auto sizes = throughput::tensor_argument(spec, 0) + .at("sizes") + .get>(); + return double(sizes.at(0)) * (2.0 * sizes.at(1) + 15); +} +REGISTER_THROUGHPUT_FUNCTIONS { + REGISTER_OPERATOR_MEMORY_TRAFFIC( + etvk.choose_qparams_per_row.default, throughput::tensor_traffic); + REGISTER_OPERATOR_COMPUTE_FLOPS( + etvk.choose_qparams_per_row.default, choose_qparams_flops); +} + +} // namespace +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/quantization/choose_qparams_per_row/cases/correctness.json b/backends/vulkan/test/ops/benchmark_v2/operators/quantization/choose_qparams_per_row/cases/correctness.json new file mode 100644 index 00000000000..5e9fca4c228 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/quantization/choose_qparams_per_row/cases/correctness.json @@ -0,0 +1,564 @@ +[ + { + "name": "choose_qparams_per_row correctness [4, 16]", + "operator": "etvk.choose_qparams_per_row.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_INPUT_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4, 16], + "dtype": "float32", + "storage": "v_INPUT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int": -128 + }, + { + "int": 127 + }, + { + "tensor": { + "name": "scale", + "sizes": [4], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + }, + { + "tensor": { + "name": "zero_point", + "sizes": [4], + "dtype": "int8", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "choose_qparams_per_row correctness [8, 32]", + "operator": "etvk.choose_qparams_per_row.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_INPUT_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [8, 32], + "dtype": "float32", + "storage": "v_INPUT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int": -128 + }, + { + "int": 127 + }, + { + "tensor": { + "name": "scale", + "sizes": [8], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + }, + { + "tensor": { + "name": "zero_point", + "sizes": [8], + "dtype": "int8", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "choose_qparams_per_row correctness [16, 64]", + "operator": "etvk.choose_qparams_per_row.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_INPUT_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [16, 64], + "dtype": "float32", + "storage": "v_INPUT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int": -128 + }, + { + "int": 127 + }, + { + "tensor": { + "name": "scale", + "sizes": [16], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + }, + { + "tensor": { + "name": "zero_point", + "sizes": [16], + "dtype": "int8", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "choose_qparams_per_row correctness [32, 128]", + "operator": "etvk.choose_qparams_per_row.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_INPUT_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [32, 128], + "dtype": "float32", + "storage": "v_INPUT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int": -128 + }, + { + "int": 127 + }, + { + "tensor": { + "name": "scale", + "sizes": [32], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + }, + { + "tensor": { + "name": "zero_point", + "sizes": [32], + "dtype": "int8", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "choose_qparams_per_row correctness [64, 256]", + "operator": "etvk.choose_qparams_per_row.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_INPUT_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [64, 256], + "dtype": "float32", + "storage": "v_INPUT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int": -128 + }, + { + "int": 127 + }, + { + "tensor": { + "name": "scale", + "sizes": [64], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + }, + { + "tensor": { + "name": "zero_point", + "sizes": [64], + "dtype": "int8", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "choose_qparams_per_row correctness [128, 512]", + "operator": "etvk.choose_qparams_per_row.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_INPUT_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [128, 512], + "dtype": "float32", + "storage": "v_INPUT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int": -128 + }, + { + "int": 127 + }, + { + "tensor": { + "name": "scale", + "sizes": [128], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + }, + { + "tensor": { + "name": "zero_point", + "sizes": [128], + "dtype": "int8", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "choose_qparams_per_row correctness [1,", + "operator": "etvk.choose_qparams_per_row.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_INPUT_SIZES": [ + [1, 512], + [1, 2048] + ], + "v_INPUT_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": "v_INPUT_SIZES", + "dtype": "float32", + "storage": "v_INPUT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int": -128 + }, + { + "int": 127 + }, + { + "tensor": { + "name": "scale", + "sizes": [1], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + }, + { + "tensor": { + "name": "zero_point", + "sizes": [1], + "dtype": "int8", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "choose_qparams_per_row correctness [256, 1024]", + "operator": "etvk.choose_qparams_per_row.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_INPUT_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [256, 1024], + "dtype": "float32", + "storage": "v_INPUT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int": -128 + }, + { + "int": 127 + }, + { + "tensor": { + "name": "scale", + "sizes": [256], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + }, + { + "tensor": { + "name": "zero_point", + "sizes": [256], + "dtype": "int8", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "choose_qparams_per_row correctness [512, 2048]", + "operator": "etvk.choose_qparams_per_row.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_INPUT_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [512, 2048], + "dtype": "float32", + "storage": "v_INPUT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int": -128 + }, + { + "int": 127 + }, + { + "tensor": { + "name": "scale", + "sizes": [512], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + }, + { + "tensor": { + "name": "zero_point", + "sizes": [512], + "dtype": "int8", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/quantization/choose_qparams_per_row/cases/debug.json b/backends/vulkan/test/ops/benchmark_v2/operators/quantization/choose_qparams_per_row/cases/debug.json new file mode 100644 index 00000000000..b5e7d19a524 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/quantization/choose_qparams_per_row/cases/debug.json @@ -0,0 +1,64 @@ +[ + { + "name": "choose_qparams_per_row debug [4, 8]", + "operator": "etvk.choose_qparams_per_row.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_INPUT_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [4, 8], + "dtype": "float32", + "storage": "v_INPUT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int": -128 + }, + { + "int": 127 + }, + { + "tensor": { + "name": "scale", + "sizes": [4], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + }, + { + "tensor": { + "name": "zero_point", + "sizes": [4], + "dtype": "int8", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/quantization/choose_qparams_per_row/cases/performance.json b/backends/vulkan/test/ops/benchmark_v2/operators/quantization/choose_qparams_per_row/cases/performance.json new file mode 100644 index 00000000000..fbeac1b3a1a --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/quantization/choose_qparams_per_row/cases/performance.json @@ -0,0 +1,64 @@ +[ + { + "name": "choose_qparams_per_row performance [1, 8096]", + "operator": "etvk.choose_qparams_per_row.default", + "target_ms": 100, + "warmup_runs": 1, + "benchmark_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.002, + "rtol": 0.001 + }, + "sweep_variables": { + "v_INPUT_STORAGE": ["texture3d", "buffer"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 8096], + "dtype": "float32", + "storage": "v_INPUT_STORAGE", + "layout": "width", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "int": -128 + }, + { + "int": 127 + }, + { + "tensor": { + "name": "scale", + "sizes": [1], + "dtype": "float32", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + }, + { + "tensor": { + "name": "zero_point", + "sizes": [1], + "dtype": "int8", + "storage": "texture3d", + "layout": "width", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/quantize/q8ta/Q8taQdqReference.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/quantize/q8ta/Q8taQdqReference.cpp new file mode 100644 index 00000000000..1b072e4d3e1 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/quantize/q8ta/Q8taQdqReference.cpp @@ -0,0 +1,49 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include + +namespace vkcompute::shader_profiling::reference { +static Tensor q8ta_quantize_reference( + const Json& spec, + const TensorMap& tensors, + size_t threads) { + Tensor out = tensor_arg(spec, 0, tensors); + const float s = q8ta::scale(spec, 1); + const int64_t zp = q8ta::integer(spec, 2); + const int64_t minimum = q8ta::integer(spec, 3), + maximum = q8ta::integer(spec, 4); + require( + -128 <= minimum && minimum <= maximum && maximum <= 127, + "Quantize reference requires signed int8 quantization bounds"); + parallel_for(out.values.size(), threads, [&](size_t i) { + out.values[i] = double(std::clamp( + q8ta::quantize(float(out.values[i]), s, zp), minimum, maximum)); + }); + return out; +} +static Tensor q8ta_dequantize_reference( + const Json& spec, + const TensorMap& tensors, + size_t threads) { + Tensor out = tensor_arg(spec, 0, tensors); + const float s = q8ta::scale(spec, 1); + const int64_t zp = q8ta::integer(spec, 2); + parallel_for(out.values.size(), threads, [&](size_t i) { + out.values[i] = q8ta::dequantize(int64_t(out.values[i]), s, zp); + }); + return out; +} +REGISTER_REFERENCE_FUNCTIONS { + REGISTER_REFERENCE_FN(q8ta_quantize, q8ta_quantize_reference); + REGISTER_REFERENCE_FN(q8ta_dequantize, q8ta_dequantize_reference); + REGISTER_OPERATOR_REFERENCE( + quantized_decomposed.quantize_per_tensor.default, q8ta_quantize); + REGISTER_OPERATOR_REFERENCE( + quantized_decomposed.dequantize_per_tensor.default, q8ta_dequantize); +} +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/quantize/q8ta/Q8taQdqRoutes.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/quantize/q8ta/Q8taQdqRoutes.cpp new file mode 100644 index 00000000000..b2b070b8242 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/quantize/q8ta/Q8taQdqRoutes.cpp @@ -0,0 +1,67 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include +#include + +namespace vkcompute::shader_profiling { +namespace { +void check_legacy_fp_layout(ComputeGraph& graph, ValueRef tensor) { + const auto storage = graph.storage_type_of(tensor); + const auto layout = graph.estimate_memory_layout_of(tensor); + if (!((storage == utils::kBuffer && layout == utils::kWidthPacked) || + (storage == utils::kTexture3D && layout == utils::kChannelsPacked))) { + throw UnsupportedRoute( + "Legacy 4w4c Q/DQ requires width-packed floating buffers or " + "channels-packed texture3d floating tensors"); + } +} +} // namespace + +static Json quantize_legacy_route( + ComputeGraph& graph, + const std::vector& args, + const Json& options) { + if (!options.empty()) + throw std::invalid_argument("Legacy quantize route accepts no options"); + const ValueRef output = args.back(); + if (graph.estimate_memory_layout_of(output) != utils::kPackedInt8_4W4C) { + VK_GET_OP_FN("quantized_decomposed.quantize_per_tensor.default") + (graph, args); + return {{"name", "auto"}, {"options", Json::object()}}; + } + check_legacy_fp_layout(graph, args[0]); + add_quantize_and_pack_4w4c_node(graph, args[0], args[1], args[2], output); + return {{"name", "legacy_4w4c"}, {"options", Json::object()}}; +} +static Json dequantize_legacy_route( + ComputeGraph& graph, + const std::vector& args, + const Json& options) { + if (!options.empty()) + throw std::invalid_argument("Legacy dequantize route accepts no options"); + if (graph.estimate_memory_layout_of(args[0]) != utils::kPackedInt8_4W4C) { + VK_GET_OP_FN("quantized_decomposed.dequantize_per_tensor.default") + (graph, args); + return {{"name", "auto"}, {"options", Json::object()}}; + } + check_legacy_fp_layout(graph, args.back()); + add_unpack_4w4c_and_dequantize_node( + graph, args[0], args[1], args[2], args.back()); + return {{"name", "legacy_4w4c"}, {"options", Json::object()}}; +} +REGISTER_OPERATOR_ROUTES { + REGISTER_OPERATOR_ROUTE( + quantized_decomposed.quantize_per_tensor.default, + legacy_4w4c, + quantize_legacy_route); + REGISTER_OPERATOR_ROUTE( + quantized_decomposed.dequantize_per_tensor.default, + legacy_4w4c, + dequantize_legacy_route); +} +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/quantize/q8ta/Q8taQdqThroughput.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/quantize/q8ta/Q8taQdqThroughput.cpp new file mode 100644 index 00000000000..5ef4126554c --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/quantize/q8ta/Q8taQdqThroughput.cpp @@ -0,0 +1,30 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +namespace vkcompute::shader_profiling { +namespace { +double quantize_flops(const Json& spec) { + return 2 * throughput::tensor_elements(throughput::output_tensor(spec)); +} +double dequantize_flops(const Json& spec) { + return throughput::tensor_elements(throughput::output_tensor(spec)); +} +REGISTER_THROUGHPUT_FUNCTIONS { + REGISTER_OPERATOR_MEMORY_TRAFFIC( + quantized_decomposed.quantize_per_tensor.default, + throughput::tensor_traffic); + REGISTER_OPERATOR_MEMORY_TRAFFIC( + quantized_decomposed.dequantize_per_tensor.default, + throughput::tensor_traffic); + REGISTER_OPERATOR_COMPUTE_FLOPS( + quantized_decomposed.quantize_per_tensor.default, quantize_flops); + REGISTER_OPERATOR_COMPUTE_FLOPS( + quantized_decomposed.dequantize_per_tensor.default, dequantize_flops); +} +} // namespace +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/quantize/q8ta/Q8taReferenceCommon.h b/backends/vulkan/test/ops/benchmark_v2/operators/quantize/q8ta/Q8taReferenceCommon.h new file mode 100644 index 00000000000..319b6dc5dcd --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/quantize/q8ta/Q8taReferenceCommon.h @@ -0,0 +1,90 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once +#include +#include +#include +#include + +namespace vkcompute::shader_profiling::reference::q8ta { +inline float scale(const Json& spec, size_t i) { + return spec.at("args").at(i).at("float").get(); +} +inline int64_t integer(const Json& spec, size_t i) { + return spec.at("args").at(i).at("int").get(); +} +inline int64_t requantize(float value, float inverse_scale, int64_t zp) { + const float scaled = value * inverse_scale; + if (scaled < float(-128 - zp) - 1.0f) + return -128; + if (scaled > float(127 - zp) + 1.0f) + return 127; + const float base = std::floor(scaled); + const float fraction = scaled - base; + const bool round_up = + fraction > .5f || (fraction == .5f && std::fmod(base, 2.0f) != 0.0f); + const int64_t rounded = int64_t(base) + int64_t(round_up); + return std::clamp(rounded + zp, int64_t(-128), int64_t(127)); +} +inline int64_t quantize(float value, float s, int64_t zp) { + return requantize(value, 1.0f / s, zp); +} +inline int64_t requantize_accumulator( + int64_t corrected_accumulator, + float input_scale, + float weight_scale, + float output_scale, + int64_t output_zero_point, + float bias = 0.0f, + bool relu = false, + bool fused_bias = true) { + const float combined_scale = weight_scale * input_scale; + float value = std::fma( + float(corrected_accumulator), combined_scale, fused_bias ? bias : 0.0f); + if (!fused_bias) + value += bias; + if (relu) + value = std::max(value, 0.0f); + return requantize(value, 1.0f / output_scale, output_zero_point); +} +inline float dequantize(int64_t value, float s, int64_t zp) { + return float(value - zp) * s; +} +inline Tensor quantized_input(const Tensor& input, float s, int64_t zp) { + Tensor result = input; + for (double& value : result.values) + value = double(quantize(float(value), s, zp)); + return result; +} +inline Tensor output_tensor(const Json& spec, size_t index) { + const std::string name = spec.at("args").at(index).at("tensor"); + auto sizes = + spec.at("tensors").at(name).at("sizes").get>(); + size_t count = 1; + for (int64_t size : sizes) + count *= size_t(size); + return {std::move(sizes), std::vector(count)}; +} +inline size_t broadcast_index( + size_t index, + const std::vector& out, + const std::vector& sizes) { + size_t source = 0, stride = 1; + for (size_t i = 0; i < out.size(); ++i) { + const size_t dim = out.size() - 1 - i; + const size_t coordinate = index % size_t(out[dim]); + index /= size_t(out[dim]); + if (i < sizes.size()) { + const size_t source_dim = sizes.size() - 1 - i; + if (sizes[source_dim] != 1) + source += coordinate * stride; + stride *= size_t(sizes[source_dim]); + } + } + return source; +} +} // namespace vkcompute::shader_profiling::reference::q8ta diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/quantize/q8ta/cases/correctness.json b/backends/vulkan/test/ops/benchmark_v2/operators/quantize/q8ta/cases/correctness.json new file mode 100644 index 00000000000..5fc97e12388 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/quantize/q8ta/cases/correctness.json @@ -0,0 +1,710 @@ +[ + { + "name": "q8ta/qdq/correctness quantize buffer", + "operator": "quantized_decomposed.quantize_per_tensor.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.014060718249784, + "rtol": 0 + }, + "route": "auto", + "sweep_variables": { + "v_SIZES": [ + [144], + [90], + [1, 144], + [1, 90], + [1, 4], + [3, 32], + [1, 16, 32], + [1, 3, 64], + [1, 3, 16, 16], + [1, 8, 32, 32], + [1, 16, 24, 24], + [1, 32, 12, 12], + [1, 1, 64, 64], + [1, 3, 64, 64], + [1, 4, 16, 16], + [1, 8, 20, 20], + [1, 16, 14, 14], + [1, 8, 28, 28], + [1, 3, 15, 15], + [1, 13, 31, 31], + [1, 17, 23, 23], + [1, 64, 128, 128], + [1, 32, 64, 64], + [1, 128, 56, 56], + [2, 3, 16, 16], + [2, 8, 8, 8], + [2, 4, 8, 16], + [2, 3, 8, 8] + ], + "v_FLOAT_LAYOUT": ["width", "channels"], + "v_PACKED_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": "v_SIZES", + "dtype": "float32", + "storage": "buffer", + "layout": "v_FLOAT_LAYOUT", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 0.00711200013756752 + }, + { + "int": 0 + }, + { + "int": -128 + }, + { + "int": 127 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "output", + "sizes": "v_SIZES", + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_PACKED_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/qdq/correctness quantize texture3d", + "operator": "quantized_decomposed.quantize_per_tensor.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.014060718249784, + "rtol": 0 + }, + "route": "auto", + "sweep_variables": { + "v_SIZES": [ + [144], + [90], + [1, 144], + [1, 90], + [1, 4], + [3, 32], + [1, 16, 32], + [1, 3, 64], + [1, 3, 16, 16], + [1, 8, 32, 32], + [1, 16, 24, 24], + [1, 32, 12, 12], + [1, 1, 64, 64], + [1, 3, 64, 64], + [1, 4, 16, 16], + [1, 8, 20, 20], + [1, 16, 14, 14], + [1, 8, 28, 28], + [1, 3, 15, 15], + [1, 13, 31, 31], + [1, 17, 23, 23], + [1, 64, 128, 128], + [1, 32, 64, 64], + [1, 128, 56, 56] + ], + "v_FLOAT_LAYOUT": ["width", "channels"], + "v_PACKED_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": "v_SIZES", + "dtype": "float32", + "storage": "texture3d", + "layout": "v_FLOAT_LAYOUT", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 0.00711200013756752 + }, + { + "int": 0 + }, + { + "int": -128 + }, + { + "int": 127 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "output", + "sizes": "v_SIZES", + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_PACKED_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/qdq/correctness quantize buffer legacy_4w4c", + "operator": "quantized_decomposed.quantize_per_tensor.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.014060718249784, + "rtol": 0 + }, + "route": "legacy_4w4c", + "sweep_variables": { + "v_SIZES": [ + [144], + [90], + [1, 144], + [1, 90], + [1, 4], + [3, 32], + [1, 16, 32], + [1, 3, 64], + [1, 3, 16, 16], + [1, 8, 32, 32], + [1, 16, 24, 24], + [1, 32, 12, 12], + [1, 1, 64, 64], + [1, 3, 64, 64], + [1, 4, 16, 16], + [1, 8, 20, 20], + [1, 16, 14, 14], + [1, 8, 28, 28], + [1, 3, 15, 15], + [1, 13, 31, 31], + [1, 17, 23, 23], + [1, 64, 128, 128], + [1, 32, 64, 64], + [1, 128, 56, 56] + ] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": "v_SIZES", + "dtype": "float32", + "storage": "buffer", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 0.00711200013756752 + }, + { + "int": 0 + }, + { + "int": -128 + }, + { + "int": 127 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "output", + "sizes": "v_SIZES", + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/qdq/correctness quantize texture3d legacy_4w4c", + "operator": "quantized_decomposed.quantize_per_tensor.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.014060718249784, + "rtol": 0 + }, + "route": "legacy_4w4c", + "sweep_variables": { + "v_SIZES": [ + [144], + [90], + [1, 144], + [1, 90], + [1, 4], + [3, 32], + [1, 16, 32], + [1, 3, 64], + [1, 3, 16, 16], + [1, 8, 32, 32], + [1, 16, 24, 24], + [1, 32, 12, 12], + [1, 1, 64, 64], + [1, 3, 64, 64], + [1, 4, 16, 16], + [1, 8, 20, 20], + [1, 16, 14, 14], + [1, 8, 28, 28], + [1, 3, 15, 15], + [1, 13, 31, 31], + [1, 17, 23, 23], + [1, 64, 128, 128], + [1, 32, 64, 64], + [1, 128, 56, 56] + ] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": "v_SIZES", + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 0.00711200013756752 + }, + { + "int": 0 + }, + { + "int": -128 + }, + { + "int": 127 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "output", + "sizes": "v_SIZES", + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/qdq/correctness dequantize buffer", + "operator": "quantized_decomposed.dequantize_per_tensor.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0072119999676942825, + "rtol": 0.0010000000474974513 + }, + "route": "auto", + "sweep_variables": { + "v_SIZES": [ + [144], + [90], + [1, 144], + [1, 90], + [1, 4], + [3, 32], + [1, 16, 32], + [1, 3, 64], + [1, 3, 16, 16], + [1, 8, 32, 32], + [1, 16, 24, 24], + [1, 32, 12, 12], + [1, 1, 64, 64], + [1, 3, 64, 64], + [1, 4, 16, 16], + [1, 8, 20, 20], + [1, 16, 14, 14], + [1, 8, 28, 28], + [1, 3, 15, 15], + [1, 13, 31, 31], + [1, 17, 23, 23], + [1, 64, 128, 128], + [1, 32, 64, 64], + [1, 128, 56, 56], + [2, 3, 16, 16], + [2, 8, 8, 8], + [2, 4, 8, 16], + [2, 3, 8, 8] + ], + "v_FLOAT_LAYOUT": ["width", "channels"], + "v_PACKED_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": "v_SIZES", + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_PACKED_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00711200013756752 + }, + { + "int": 0 + }, + { + "int": -128 + }, + { + "int": 127 + }, + { + "int": 2 + }, + { + "int": 10 + }, + { + "tensor": { + "name": "output", + "sizes": "v_SIZES", + "dtype": "float32", + "storage": "buffer", + "layout": "v_FLOAT_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/qdq/correctness dequantize texture3d", + "operator": "quantized_decomposed.dequantize_per_tensor.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0072119999676942825, + "rtol": 0.0010000000474974513 + }, + "route": "auto", + "sweep_variables": { + "v_SIZES": [ + [144], + [90], + [1, 144], + [1, 90], + [1, 4], + [3, 32], + [1, 16, 32], + [1, 3, 64], + [1, 3, 16, 16], + [1, 8, 32, 32], + [1, 16, 24, 24], + [1, 32, 12, 12], + [1, 1, 64, 64], + [1, 3, 64, 64], + [1, 4, 16, 16], + [1, 8, 20, 20], + [1, 16, 14, 14], + [1, 8, 28, 28], + [1, 3, 15, 15], + [1, 13, 31, 31], + [1, 17, 23, 23], + [1, 64, 128, 128], + [1, 32, 64, 64], + [1, 128, 56, 56] + ], + "v_FLOAT_LAYOUT": ["width", "channels"], + "v_PACKED_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": "v_SIZES", + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_PACKED_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00711200013756752 + }, + { + "int": 0 + }, + { + "int": -128 + }, + { + "int": 127 + }, + { + "int": 2 + }, + { + "int": 10 + }, + { + "tensor": { + "name": "output", + "sizes": "v_SIZES", + "dtype": "float32", + "storage": "texture3d", + "layout": "v_FLOAT_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/qdq/correctness dequantize buffer legacy_4w4c", + "operator": "quantized_decomposed.dequantize_per_tensor.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0072119999676942825, + "rtol": 0.0010000000474974513 + }, + "route": "legacy_4w4c", + "sweep_variables": { + "v_SIZES": [ + [144], + [90], + [1, 144], + [1, 90], + [1, 4], + [3, 32], + [1, 16, 32], + [1, 3, 64], + [1, 3, 16, 16], + [1, 8, 32, 32], + [1, 16, 24, 24], + [1, 32, 12, 12], + [1, 1, 64, 64], + [1, 3, 64, 64], + [1, 4, 16, 16], + [1, 8, 20, 20], + [1, 16, 14, 14], + [1, 8, 28, 28], + [1, 3, 15, 15], + [1, 13, 31, 31], + [1, 17, 23, 23], + [1, 64, 128, 128], + [1, 32, 64, 64], + [1, 128, 56, 56] + ] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": "v_SIZES", + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00711200013756752 + }, + { + "int": 0 + }, + { + "int": -128 + }, + { + "int": 127 + }, + { + "int": 2 + }, + { + "int": 10 + }, + { + "tensor": { + "name": "output", + "sizes": "v_SIZES", + "dtype": "float32", + "storage": "buffer", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/qdq/correctness dequantize texture3d legacy_4w4c", + "operator": "quantized_decomposed.dequantize_per_tensor.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0072119999676942825, + "rtol": 0.0010000000474974513 + }, + "route": "legacy_4w4c", + "sweep_variables": { + "v_SIZES": [ + [144], + [90], + [1, 144], + [1, 90], + [1, 4], + [3, 32], + [1, 16, 32], + [1, 3, 64], + [1, 3, 16, 16], + [1, 8, 32, 32], + [1, 16, 24, 24], + [1, 32, 12, 12], + [1, 1, 64, 64], + [1, 3, 64, 64], + [1, 4, 16, 16], + [1, 8, 20, 20], + [1, 16, 14, 14], + [1, 8, 28, 28], + [1, 3, 15, 15], + [1, 13, 31, 31], + [1, 17, 23, 23], + [1, 64, 128, 128], + [1, 32, 64, 64], + [1, 128, 56, 56] + ] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": "v_SIZES", + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00711200013756752 + }, + { + "int": 0 + }, + { + "int": -128 + }, + { + "int": 127 + }, + { + "int": 2 + }, + { + "int": 10 + }, + { + "tensor": { + "name": "output", + "sizes": "v_SIZES", + "dtype": "float32", + "storage": "texture3d", + "layout": "channels", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/quantize/q8ta/cases/debug.json b/backends/vulkan/test/ops/benchmark_v2/operators/quantize/q8ta/cases/debug.json new file mode 100644 index 00000000000..20f670d615f --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/quantize/q8ta/cases/debug.json @@ -0,0 +1,274 @@ +[ + { + "name": "q8ta/qdq/debug float32 buffer auto", + "operator": "quantized_decomposed.quantize_per_tensor.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.014060718249784, + "rtol": 0 + }, + "route": "auto", + "sweep_variables": { + "v_INPUT_LAYOUT": ["width", "channels"], + "v_SIZES": [ + [1, 16, 16, 16], + [1, 144], + [1, 90], + [2, 3, 16, 16], + [2, 4, 8, 16] + ], + "v_OUTPUT_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": "v_SIZES", + "dtype": "float32", + "storage": "buffer", + "layout": "v_INPUT_LAYOUT", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 0.00711200013756752 + }, + { + "int": 0 + }, + { + "int": -128 + }, + { + "int": 127 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "output", + "sizes": "v_SIZES", + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_OUTPUT_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/qdq/debug float32 buffer legacy_4w4c", + "operator": "quantized_decomposed.quantize_per_tensor.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.014060718249784, + "rtol": 0 + }, + "route": "legacy_4w4c", + "sweep_variables": { + "v_INPUT_LAYOUT": ["width", "channels"], + "v_SIZES": [ + [1, 16, 16, 16], + [1, 144], + [1, 90] + ] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": "v_SIZES", + "dtype": "float32", + "storage": "buffer", + "layout": "v_INPUT_LAYOUT", + "data": { + "rand_uniform_float": [-1, 1] + } + } + }, + { + "float": 0.00711200013756752 + }, + { + "int": 0 + }, + { + "int": -128 + }, + { + "int": 127 + }, + { + "int": 2 + }, + { + "tensor": { + "name": "output", + "sizes": "v_SIZES", + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/qdq/debug int8x4 buffer auto", + "operator": "quantized_decomposed.dequantize_per_tensor.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0072119999676942825, + "rtol": 0.0010000000474974513 + }, + "route": "auto", + "sweep_variables": { + "v_INPUT_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"], + "v_SIZES": [ + [1, 16, 16, 16], + [1, 144], + [1, 90], + [2, 3, 16, 16], + [2, 4, 8, 16] + ], + "v_OUTPUT_LAYOUT": ["width", "channels"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": "v_SIZES", + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_INPUT_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00711200013756752 + }, + { + "int": 0 + }, + { + "int": -128 + }, + { + "int": 127 + }, + { + "int": 2 + }, + { + "int": 10 + }, + { + "tensor": { + "name": "output", + "sizes": "v_SIZES", + "dtype": "float32", + "storage": "buffer", + "layout": "v_OUTPUT_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + }, + { + "name": "q8ta/qdq/debug int8x4 buffer 4w4c legacy_4w4c", + "operator": "quantized_decomposed.dequantize_per_tensor.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 0.0072119999676942825, + "rtol": 0.0010000000474974513 + }, + "route": "legacy_4w4c", + "sweep_variables": { + "v_SIZES": [ + [1, 16, 16, 16], + [1, 144], + [1, 90] + ], + "v_OUTPUT_LAYOUT": ["width", "channels"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": "v_SIZES", + "dtype": "int8x4", + "storage": "buffer", + "layout": "4w4c", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00711200013756752 + }, + { + "int": 0 + }, + { + "int": -128 + }, + { + "int": 127 + }, + { + "int": 2 + }, + { + "int": 10 + }, + { + "tensor": { + "name": "output", + "sizes": "v_SIZES", + "dtype": "float32", + "storage": "buffer", + "layout": "v_OUTPUT_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/quantize/q8ta/test/q8_benchmark_reference_test.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/quantize/q8ta/test/q8_benchmark_reference_test.cpp new file mode 100644 index 00000000000..bdab0110e8b --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/quantize/q8ta/test/q8_benchmark_reference_test.cpp @@ -0,0 +1,333 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include +#include +#include +#include +namespace vkcompute::shader_profiling { +namespace { +Json tensor_arg( + const char* name, + const Json& sizes, + const Json& data, + const char* dtype = "float32", + const char* layout = "width") { + return { + {"tensor", + {{"name", name}, + {"sizes", sizes}, + {"dtype", dtype}, + {"layout", layout}, + {"storage", + std::string(name) == "input" || std::string(name) == "output" || + std::string(name) == "other" + ? "buffer" + : "constant"}, + {"data", data}}}}; +} +std::vector compute(const char* op, const Json& args) { + const Json spec = + parse_configs({{"operator", op}, {"args_sweep", Json::array({args})}}) + .front(); + auto data = make_tensor_data(spec.at("tensors")); + data.erase("output"); + return reference_outputs(spec, {}, data, 2).at("output"); +} +TEST(Q8BenchmarkReference, AddBroadcastsQuantizedInputsAndHonorsAlpha) { + const Json args = Json::array( + {tensor_arg("input", {2, 1}, {2, 4}, "int8x4", "4w"), + tensor_arg("other", {2}, {-2, 4}, "int8x4", "4w"), + {{"float", .5}}, + {{"int", 0}}, + {{"float", .25}}, + {{"int", 0}}, + {{"float", .5}}, + {{"int", 0}}, + {{"float", 2}}, + tensor_arg("output", {2, 2}, {{"fill", 0}}, "int8x4", "4w")}); + EXPECT_EQ( + compute("et_vk.q8ta_add.default", args), + (std::vector{0, 6, 2, 8})); +} +TEST(Q8BenchmarkReference, AddAlphaAndSaturationWithBroadcasting) { + Json args = Json::array( + {tensor_arg("input", {2, 1}, {120, -120}, "int8x4", "4w"), + tensor_arg("other", {2}, {-20, 20}, "int8x4", "4w"), + {{"float", 1}}, + {{"int", 0}}, + {{"float", 1}}, + {{"int", 0}}, + {{"float", 1}}, + {{"int", 0}}, + {{"float", 0}}, + tensor_arg("output", {2, 2}, {{"fill", 0}}, "int8x4", "4w")}); + EXPECT_EQ( + compute("et_vk.q8ta_add.default", args), + (std::vector{120, 120, -120, -120})); + args[8]["float"] = .5; + EXPECT_EQ( + compute("et_vk.q8ta_add.default", args), + (std::vector{110, 127, -128, -110})); + args[8]["float"] = -1; + EXPECT_EQ( + compute("et_vk.q8ta_add.default", args), + (std::vector{127, 100, -100, -128})); +} +TEST(Q8BenchmarkReference, ClonePreservesSignedBytesAcrossLayouts) { + const Json args = Json::array( + {tensor_arg("input", {5}, {-128, -1, 0, 1, 127}, "int8x4", "4w"), + nullptr, + tensor_arg("output", {5}, {{"fill", 0}}, "int8x4", "4c1w")}); + EXPECT_EQ( + compute("aten.clone.default", args), + (std::vector{-128, -1, 0, 1, 127})); +} +TEST(Q8BenchmarkReference, QuantizeRoundsAndClampsIntegerOutput) { + const Json args = Json::array( + {tensor_arg("input", {5}, {-100, -.26, 0, .26, 100}), + {{"float", .5}}, + {{"int", 0}}, + {{"int", -128}}, + {{"int", 127}}, + {{"int", 2}}, + tensor_arg("output", {5}, {{"fill", 0}}, "int8x4", "4w")}); + EXPECT_EQ( + compute("quantized_decomposed.quantize_per_tensor.default", args), + (std::vector{-128, -1, 0, 1, 127})); +} +TEST(Q8BenchmarkReference, DequantizeAppliesScaleAndZeroPoint) { + const Json args = Json::array( + {tensor_arg("input", {5}, {-128, -1, 0, 1, 127}, "int8x4", "4w"), + {{"float", .5}}, + {{"int", -2}}, + {{"int", -128}}, + {{"int", 127}}, + {{"int", 2}}, + {{"int", 10}}, + tensor_arg("output", {5}, {{"fill", 0}})}); + EXPECT_EQ( + compute("quantized_decomposed.dequantize_per_tensor.default", args), + (std::vector{-63, .5, 1, 1.5, 64.5})); +} +TEST(Q8BenchmarkReference, ReluUsesQuantizedZeroPoints) { + const Json args = Json::array( + {tensor_arg("input", {4}, {-4, -2, 0, 2}, "int8x4", "4w"), + {{"float", 1}}, + {{"int", -2}}, + {{"float", .5}}, + {{"int", 3}}, + tensor_arg("output", {4}, {{"fill", 0}}, "int8x4", "4w")}); + EXPECT_EQ( + compute("et_vk.q8ta_relu.default", args), + (std::vector{3, 3, 7, 11})); +} +TEST(Q8BenchmarkReference, PixelShufflePermutesQuantizedBytes) { + const Json args = Json::array( + {tensor_arg("input", {1, 4, 1, 1}, {1, 2, 3, 4}, "int8x4", "4w4c"), + {{"float", 1}}, + {{"int", 0}}, + {{"float", 1}}, + {{"int", 0}}, + {{"int", 2}}, + tensor_arg("output", {1, 1, 2, 2}, {{"fill", 0}}, "int8x4", "4c1w")}); + EXPECT_EQ( + compute("et_vk.q8ta_pixel_shuffle.default", args), + (std::vector{1, 2, 3, 4})); +} +TEST(Q8BenchmarkReference, PixelShuffleRequantizesUsingInverseOutputScale) { + const Json args = Json::array( + {tensor_arg("input", {1, 4, 1, 1}, {-128, -1, 1, 127}, "int8x4", "4w4c"), + {{"float", .5}}, + {{"int", -1}}, + {{"float", 2}}, + {{"int", 3}}, + {{"int", 2}}, + tensor_arg("output", {1, 1, 2, 2}, {{"fill", 0}}, "int8x4", "4c1w")}); + EXPECT_EQ( + compute("et_vk.q8ta_pixel_shuffle.default", args), + (std::vector{-124, 3, 5, 127})); +} +Json convolution_args( + const Json& input, + const Json& weight, + const Json& scales, + const Json& bias, + const Json& output) { + return Json::array( + {input, + {{"float", 1}}, + {{"int", 0}}, + weight, + tensor_arg("sums", {4}, {0, 0, 0, 0}, "int32"), + scales, + {{"float", 1}}, + {{"int", 0}}, + bias, + {{"int_list", {1, 1}}}, + {{"int_list", {1, 1}}}, + {{"int_list", {0, 0}}}, + {{"int_list", {1, 1}}}, + {{"int", 1}}, + {{"string", "none"}}, + output}); +} +TEST(Q8BenchmarkReference, ConvolutionIgnoresPaddedWeightAndChannelParameters) { + const Json args = convolution_args( + tensor_arg("input", {1, 3, 1, 1}, {1, 2, 3}, "int8x4", "4c"), + tensor_arg("weight", {2, 4}, {1, 2, 3, 99, 4, 5, 6, -99}, "int8"), + tensor_arg("scales", {4}, {1, 1, 99, 99}), + tensor_arg("bias", {4}, {0, 0, 99, 99}), + tensor_arg("output", {1, 2, 1, 1}, {{"fill", 0}}, "int8x4", "4c")); + EXPECT_EQ( + compute("et_vk.q8ta_conv2d.default", args), + (std::vector{14, 32})); +} +TEST(Q8BenchmarkReference, DepthwiseReadsHeightWidthChannelWeightOrder) { + Json args = convolution_args( + tensor_arg("input", {1, 2, 1, 2}, {1, 2, 3, 4}, "int8x4", "4c"), + tensor_arg("weight", {1, 1, 2}, {2, -1}, "int8"), + tensor_arg("scales", {2}, {1, 1}), + tensor_arg("bias", {2}, {0, 0}), + tensor_arg("output", {1, 2, 1, 2}, {{"fill", 0}}, "int8x4", "4c")); + args[13]["int"] = 2; + EXPECT_EQ( + compute("et_vk.q8ta_conv2d_dw.default", args), + (std::vector{2, 4, -3, -4})); +} +TEST(Q8BenchmarkReference, LinearUsesIntegerInputsAndWeightSumCorrection) { + const Json args = Json::array( + {tensor_arg("input", {1, 4}, {4, 2, 3, 1}, "int8x4", "4h4w"), + {{"float", 1}}, + {{"int", 1}}, + tensor_arg("weight", {2, 4}, {1, -1, 0, 0, 2, 0, -1, 0}, "int8"), + tensor_arg("sums", {2}, {0, 1}, "int32"), + tensor_arg("scales", {2}, {1, 1}), + {{"float", 1}}, + {{"int", 0}}, + tensor_arg("bias", {2}, {1, -1}), + {{"string", "none"}}, + tensor_arg("output", {1, 2}, {{"fill", 0}}, "int8x4", "4h4w")}); + EXPECT_EQ( + compute("et_vk.q8ta_linear.default", args), (std::vector{3, 3})); +} +TEST(Q8BenchmarkReference, TransposedConvolutionExpandsStridedKernel) { + Json args = convolution_args( + tensor_arg("input", {1, 1, 1, 1}, {2}, "int8x4", "4c1w"), + tensor_arg("weight", {1, 4}, {1, 2, 3, 4}, "int8"), + tensor_arg("scales", {1}, {1}), + nullptr, + tensor_arg("output", {1, 1, 2, 2}, {{"fill", 0}}, "int8x4", "4c1w")); + args[9]["int_list"] = {2, 2}; + args[10]["int_list"] = {2, 2}; + args.insert(args.begin() + 12, Json{{"int_list", {0, 0}}}); + EXPECT_EQ( + compute("et_vk.q8ta_conv2d_transposed.default", args), + (std::vector{2, 4, 6, 8})); +} +TEST( + Q8BenchmarkReference, + AccumulatorRequantizationHandlesBiasReluAndClamping) { + EXPECT_EQ( + reference::q8ta::requantize_accumulator(7, .5f, 1.0f, 1.0f, -3, .5f), 1); + EXPECT_EQ( + reference::q8ta::requantize_accumulator( + -8, .5f, 1.0f, 1.0f, 4, 0.0f, true), + 4); + EXPECT_EQ( + reference::q8ta::requantize_accumulator(10000, 1.0f, 1.0f, 1.0f, 0), 127); + EXPECT_EQ( + reference::q8ta::requantize_accumulator(-10000, 1.0f, 1.0f, 1.0f, 0), + -128); +} +TEST( + Q8BenchmarkReference, + NativeLinearThroughputCountsIntegerTensorTrafficAndMacs) { + const Json args = Json::array( + {tensor_arg("input", {1, 4}, {1, 2, 3, 4}, "int8x4", "4h4w"), + {{"float", 1}}, + {{"int", 0}}, + tensor_arg("weight", {2, 4}, {1, 1, 1, 1, 1, 1, 1, 1}, "int8"), + tensor_arg("sums", {2}, {4, 4}, "int32"), + tensor_arg("scales", {2}, {1, 1}), + {{"float", 1}}, + {{"int", 0}}, + nullptr, + {{"string", "none"}}, + tensor_arg("output", {1, 2}, {{"fill", 0}}, "int8x4", "4h4w")}); + const Json spec = parse_configs({{"operator", "et_vk.q8ta_linear.default"}, + {"args_sweep", Json::array({args})}}) + .front(); + const auto work = estimate_work( + Json(operator_report(spec, Json::object(), {{"output", "write"}})), + true, + true); + ASSERT_TRUE(work.memory_bytes.has_value()); + ASSERT_TRUE(work.compute_flops.has_value()); + EXPECT_EQ(*work.memory_bytes, 30); + EXPECT_EQ(*work.compute_flops, 16); +} +TEST( + Q8BenchmarkReference, + QuantizationUsesReciprocalAndRoundsTiesBeforeZeroPoint) { + EXPECT_EQ( + reference::q8ta::quantize(-0.17499999702f, 0.070000000298f, -3), -5); + EXPECT_EQ(reference::q8ta::quantize(0.17499999702f, 0.070000000298f, 3), 5); + EXPECT_EQ( + reference::q8ta::quantize(-0.324000000954f, 0.00800000037998f, -2), -42); + const std::vector values{ + .5f, 1.5f, 2.5f, 3.5f, -.5f, -1.5f, -2.5f, -3.5f}; + const std::vector expected{3, 5, 5, 7, 3, 1, 1, -1}; + for (size_t i = 0; i < values.size(); ++i) + EXPECT_EQ(reference::q8ta::quantize(values[i], 1.0f, 3), expected[i]); +} +TEST( + Q8BenchmarkReference, + ActivationQuantizedConvolutionAppliesScaleAfterIntegerDot) { + const float scale = .07f, weight_scale = .013f; + const Json args = Json::array( + {tensor_arg("input", {1, 1, 1, 1}, {-0.17499999702f}), + {{"float", scale}}, + {{"int", -3}}, + tensor_arg("weight", {1, 4}, {127, 0, 0, 0}, "int8"), + tensor_arg("sums", {1}, {127}, "int32"), + tensor_arg("scales", {1}, {weight_scale}), + tensor_arg("bias", {1}, {.5}), + {{"int_list", {1, 1}}}, + {{"int_list", {1, 1}}}, + {{"int_list", {0, 0}}}, + {{"int_list", {1, 1}}}, + {{"int", 1}}, + tensor_arg("output", {1, 1, 1, 1}, {{"fill", 0}})}); + const float expected = float(-2 * 127) * (scale * weight_scale) + .5f; + EXPECT_EQ( + compute("et_vk.conv2d_q8ta_q8csw.default", args), + (std::vector{expected})); +} +TEST( + Q8BenchmarkReference, + ActivationQuantizedLinearCancelsIntegersBeforeScaling) { + constexpr int64_t k = 2048; + Json values = Json::array_t(size_t(k), Json(.07f)); + Json weights = Json::array_t(size_t(k), Json(-127)); + for (size_t i = 0; i < size_t(k) / 2; ++i) + weights[i] = 127; + const Json args = Json::array( + {tensor_arg("input", {1, k}, values), + {{"float", .07f}}, + {{"int", 0}}, + tensor_arg("weight", {1, k}, weights, "int8"), + tensor_arg("sums", {1}, {0}, "int32"), + tensor_arg("scales", {1}, {.013f}), + tensor_arg("bias", {1}, {.5}), + tensor_arg("output", {1, 1}, {{"fill", 0}})}); + EXPECT_EQ( + compute("et_vk.linear_q8ta_q8csw.default", args), + (std::vector{.5})); +} +} // namespace +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/unary/q8ta/Q8taUnaryReference.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/unary/q8ta/Q8taUnaryReference.cpp new file mode 100644 index 00000000000..7d16feea0c8 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/unary/q8ta/Q8taUnaryReference.cpp @@ -0,0 +1,30 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include + +namespace vkcompute::shader_profiling::reference { +static Tensor q8ta_relu_reference( + const Json& spec, + const TensorMap& tensors, + size_t threads) { + Tensor out = tensor_arg(spec, 0, tensors); + const float in_scale = q8ta::scale(spec, 1), out_scale = q8ta::scale(spec, 3); + const int64_t in_zp = q8ta::integer(spec, 2), out_zp = q8ta::integer(spec, 4); + parallel_for(out.values.size(), threads, [&](size_t i) { + const float value = + q8ta::dequantize(int64_t(out.values[i]), in_scale, in_zp); + out.values[i] = + double(q8ta::quantize(std::max(value, 0.0f), out_scale, out_zp)); + }); + return out; +} +REGISTER_REFERENCE_FUNCTIONS { + REGISTER_REFERENCE_FN(q8ta_relu, q8ta_relu_reference); + REGISTER_OPERATOR_REFERENCE(et_vk.q8ta_relu.default, q8ta_relu); +} +} // namespace vkcompute::shader_profiling::reference diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/unary/q8ta/Q8taUnaryThroughput.cpp b/backends/vulkan/test/ops/benchmark_v2/operators/unary/q8ta/Q8taUnaryThroughput.cpp new file mode 100644 index 00000000000..97a52bfa5a8 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/unary/q8ta/Q8taUnaryThroughput.cpp @@ -0,0 +1,20 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +namespace vkcompute::shader_profiling { +namespace { +double relu_flops(const Json& spec) { + return 3 * throughput::tensor_elements(throughput::output_tensor(spec)); +} +REGISTER_THROUGHPUT_FUNCTIONS { + REGISTER_OPERATOR_MEMORY_TRAFFIC( + et_vk.q8ta_relu.default, throughput::tensor_traffic); + REGISTER_OPERATOR_COMPUTE_FLOPS(et_vk.q8ta_relu.default, relu_flops); +} +} // namespace +} // namespace vkcompute::shader_profiling diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/unary/q8ta/cases/correctness.json b/backends/vulkan/test/ops/benchmark_v2/operators/unary/q8ta/cases/correctness.json new file mode 100644 index 00000000000..1a21215d212 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/unary/q8ta/cases/correctness.json @@ -0,0 +1,115 @@ +[ + { + "name": "q8ta/unary/correctness", + "operator": "et_vk.q8ta_relu.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.014060718249784, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"], + "v_SIZES": [ + [1, 3, 16, 16], + [1, 8, 32, 32], + [1, 16, 24, 24], + [1, 32, 12, 12], + [1, 1, 64, 64], + [1, 3, 64, 64], + [1, 4, 16, 16], + [1, 8, 20, 20], + [1, 16, 14, 14], + [1, 8, 28, 28], + [1, 3, 15, 15], + [1, 13, 31, 31], + [1, 17, 23, 23], + [1, 64, 128, 128], + [1, 32, 64, 64], + [1, 128, 56, 56], + [1, 128, 128, 128] + ] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": "v_SIZES", + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00711200013756752 + }, + { + "int": 0 + }, + { + "float": 0.00711200013756752 + }, + { + "int": 0 + }, + { + "tensor": { + "name": "output", + "sizes": "v_SIZES", + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": "v_SIZES", + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00711200013756752 + }, + { + "int": 0 + }, + { + "float": 0.00711200013756752 + }, + { + "int": 0 + }, + { + "tensor": { + "name": "output", + "sizes": "v_SIZES", + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/operators/unary/q8ta/cases/debug.json b/backends/vulkan/test/ops/benchmark_v2/operators/unary/q8ta/cases/debug.json new file mode 100644 index 00000000000..dd2b1ad1d1a --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/operators/unary/q8ta/cases/debug.json @@ -0,0 +1,96 @@ +[ + { + "name": "q8ta/unary/debug", + "operator": "et_vk.q8ta_relu.default", + "target_ms": 100, + "benchmark_runs": 1, + "warmup_runs": 1, + "max_dispatches": 200, + "validation_method": { + "function": "is_close", + "atol": 1.014060718249784, + "rtol": 0 + }, + "sweep_variables": { + "v_LAYOUT": ["4w", "4c", "4w4c", "4h4w", "4c1w"] + }, + "args_sweep": [ + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00711200013756752 + }, + { + "int": 0 + }, + { + "float": 0.00711200013756752 + }, + { + "int": 0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ], + [ + { + "tensor": { + "name": "input", + "sizes": [1, 16, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "rand_uniform_int": [-128, 127] + } + } + }, + { + "float": 0.00711200013756752 + }, + { + "int": 0 + }, + { + "float": 0.00711200013756752 + }, + { + "int": 0 + }, + { + "tensor": { + "name": "output", + "sizes": [1, 16, 16, 16], + "dtype": "int8x4", + "storage": "buffer", + "layout": "v_LAYOUT", + "data": { + "fill": 0 + } + } + } + ] + ] + } +] diff --git a/backends/vulkan/test/ops/benchmark_v2/targets.bzl b/backends/vulkan/test/ops/benchmark_v2/targets.bzl new file mode 100644 index 00000000000..278873c9807 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark_v2/targets.bzl @@ -0,0 +1,358 @@ +load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime") +load("@fbsource//xplat/executorch/backends/vulkan:targets.bzl", "get_platforms", "vulkan_spv_shader_lib") + +def define_common_targets(is_fbcode = False): + if is_fbcode: + return + + runtime.filegroup( + name = "migration_shaders", + srcs = native.glob(["operators/gemm/linear/qw/q4gsw/glsl/**/*"]), + visibility = ["PUBLIC"], + ) + + vulkan_spv_shader_lib( + name = "migration_shaderlib", + spv_filegroups = {":migration_shaders": "operators"}, + is_fbcode = is_fbcode, + ) + + runtime.cxx_library( + name = "config", + srcs = [ + "framework/Config.cpp", + "framework/DataInitializer.cpp", + "framework/Sweep.cpp", + ], + exported_headers = [ + "framework/Config.h", + "framework/DataInitializer.h", + "framework/Sweep.h", + ], + platforms = get_platforms(), + exported_external_deps = ["nlohmann_json"], + visibility = ["PUBLIC"], + ) + + runtime.cxx_library( + name = "throughput", + srcs = [ + "framework/ThroughputRegistry.cpp", + "operators/binary/add/AddThroughput.cpp", + "operators/binary/q8ta/Q8taBinaryThroughput.cpp", + "operators/clone/q8ta/Q8taCloneThroughput.cpp", + "operators/gemm/linear/q8ta/Q8taLinearThroughput.cpp", + "operators/pixel_shuffle/q8ta/Q8taPixelShuffleThroughput.cpp", + "operators/quantize/q8ta/Q8taQdqThroughput.cpp", + "operators/unary/q8ta/Q8taUnaryThroughput.cpp", + "operators/attention/sdpa/SdpaThroughput.cpp", + "operators/gemm/matmul/MatmulThroughput.cpp", + "operators/quantization/choose_qparams_per_row/ChooseQparamsThroughput.cpp", + "operators/gemm/linear/qw/dq8ca_q4gsw/Dq8caQ4gswThroughput.cpp", + "operators/binary/BinaryThroughput.cpp", + "operators/gemm/GemmThroughputCommon.cpp", + "operators/gemm/linear/LinearThroughput.cpp", + "operators/gemm/linear/qw/q4gsw/Q4gswLinearThroughput.cpp", + "operators/convolution/Conv2dThroughputCommon.cpp", + "operators/convolution/Conv2dThroughput.cpp", + "operators/convolution/q8ta/Q8taConv2dThroughput.cpp", + ], + exported_headers = [ + "framework/ThroughputRegistry.h", + "operators/convolution/Conv2dThroughputCommon.h", + "operators/gemm/GemmThroughputCommon.h", + ], + platforms = get_platforms(), + exported_deps = [":config"], + # @lint-ignore BUCKLINT: Avoid `link_whole=True` + link_whole = True, + visibility = ["PUBLIC"], + ) + + runtime.cxx_test( + name = "throughput_registry_test", + srcs = ["framework/test/throughput_registry_test.cpp"], + platforms = get_platforms(), + deps = [":throughput", "//third-party/googletest:gtest_main"], + ) + + runtime.cxx_test( + name = "throughput_report_test", + srcs = ["framework/test/throughput_report_test.cpp"], + platforms = get_platforms(), + deps = [":report", "//third-party/googletest:gtest_main"], + ) + + runtime.cxx_library( + name = "report", + srcs = ["framework/Report.cpp"], + exported_headers = ["framework/Report.h"], + platforms = get_platforms(), + exported_deps = [":config", ":throughput"], + visibility = ["PUBLIC"], + ) + + runtime.cxx_test( + name = "report_test", + srcs = ["framework/test/report_test.cpp"], + platforms = get_platforms(), + deps = [ + ":report", + "//third-party/googletest:gtest_main", + ], + ) + + runtime.cxx_library( + name = "reference", + srcs = [ + "framework/Reference.cpp", + "framework/ReferenceRegistry.cpp", + "operators/binary/add/AddReference.cpp", + "operators/convolution/conv1d/Conv1dReference.cpp", + "operators/gemm/matmul/MatmulReference.cpp", + "operators/embedding/qw/q4gsw/EmbeddingReference.cpp", + "operators/gemm/linear/qw/dq8ca_q4gsw/Dq8caQ4gswReference.cpp", + "operators/attention/sdpa/SdpaReference.cpp", + "operators/quantization/choose_qparams_per_row/ChooseQparamsReference.cpp", + "operators/binary/q8ta/Q8taBinaryReference.cpp", + "operators/clone/q8ta/Q8taCloneReference.cpp", + "operators/convolution/q8ta/transposed/Q8taTransposedReference.cpp", + "operators/convolution/qw/q8csw/Q8cswConv2dReference.cpp", + "operators/gemm/linear/q8ta/Q8taLinearReference.cpp", + "operators/gemm/linear/qw/q8csw/Q8cswLinearReference.cpp", + "operators/pixel_shuffle/q8ta/Q8taPixelShuffleReference.cpp", + "operators/quantize/q8ta/Q8taQdqReference.cpp", + "operators/unary/q8ta/Q8taUnaryReference.cpp", + "operators/binary/BinaryReference.cpp", + "operators/convolution/Conv2dReferenceCommon.cpp", + "operators/convolution/Conv2dReference.cpp", + "operators/convolution/q8ta/Q8taConv2dReference.cpp", + "operators/gemm/GemmReference.cpp", + "operators/gemm/linear/LinearReference.cpp", + "operators/gemm/linear/qw/q4gsw/Q4gswLinearReference.cpp", + ], + exported_headers = [ + "framework/Reference.h", + "framework/ReferenceRegistry.h", + "operators/binary/add/AddReference.h", + "operators/gemm/matmul/MatmulReference.h", + "operators/embedding/qw/q4gsw/EmbeddingReference.h", + "operators/gemm/linear/qw/dq8ca_q4gsw/Dq8caQ4gswReference.h", + "operators/attention/sdpa/SdpaReference.h", + "operators/convolution/conv1d/Conv1dReference.h", + "operators/quantize/q8ta/Q8taReferenceCommon.h", + "operators/quantization/choose_qparams_per_row/ChooseQparamsReference.h", + "operators/binary/BinaryReference.h", + "operators/convolution/Conv2dReferenceCommon.h", + "operators/convolution/Conv2dReference.h", + "operators/convolution/q8ta/Q8taConv2dReference.h", + "operators/gemm/GemmReference.h", + "operators/gemm/linear/LinearReference.h", + "operators/gemm/linear/qw/q4gsw/Q4gswLinearReference.h", + ], + compiler_flags = ["-O3"], + # Static reference registrations must survive archive dead stripping. + # @lint-ignore BUCKLINT: Avoid `link_whole=True` + link_whole = True, + platforms = get_platforms(), + exported_deps = [ + ":report", + "//executorch/runtime/core/exec_aten:lib", + ], + visibility = ["PUBLIC"], + ) + + runtime.cxx_test( + name = "reference_test", + srcs = ["framework/test/reference_test.cpp"], + platforms = get_platforms(), + deps = [ + ":reference", + "//third-party/googletest:gtest_main", + ], + ) + + runtime.cxx_test( + name = "reference_registry_test", + srcs = ["framework/test/reference_registry_test.cpp"], + platforms = get_platforms(), + deps = [ + ":reference", + "//third-party/googletest:gtest_main", + ], + ) + + runtime.cxx_library( + name = "routes", + srcs = [ + "framework/RouteRegistry.cpp", + "operators/convolution/Conv2dRouteCommon.cpp", + "operators/convolution/Conv2dRoutes.cpp", + "operators/convolution/q8ta/Q8taConv2dRoutes.cpp", + "operators/gemm/matmul/MatmulRoutes.cpp", + "operators/convolution/conv2d/DepthwiseRoutes.cpp", + "operators/gemm/linear/qw/q4gsw/Q4gswRoutes.cpp", + "operators/attention/sdpa/SdpaRoutes.cpp", + "operators/gemm/linear/q8ta/Q8taLinearRoutes.cpp", + "operators/quantize/q8ta/Q8taQdqRoutes.cpp", + ], + exported_headers = [ + "framework/RouteRegistry.h", + "operators/convolution/Conv2dRouteCommon.h", + "operators/convolution/Conv2dRoutes.h", + "operators/convolution/q8ta/Q8taConv2dRoutes.h", + ], + platforms = get_platforms(), + exported_deps = [":config", ":migration_shaderlib", "//executorch/backends/vulkan:vulkan_graph_runtime"], + # @lint-ignore BUCKLINT: Avoid `link_whole=True` + link_whole = True, + visibility = ["PUBLIC"], + ) + + runtime.cxx_test( + name = "route_registry_test", + srcs = ["framework/test/route_registry_test.cpp"], + platforms = get_platforms(), + deps = [":routes", "//third-party/googletest:gtest_main"], + ) + + runtime.cxx_test( + name = "operator_route_test", + srcs = ["framework/test/operator_route_test.cpp"], + platforms = get_platforms(), + deps = [ + "//executorch/backends/vulkan/test/api_abstractions:test_utils", + ":routes", + "//executorch/runtime/core/exec_aten:lib", + "//third-party/googletest:gtest_main", + ], + ) + + runtime.cxx_library( + name = "gpu_clock", + srcs = ["framework/GpuClock.cpp"], + exported_headers = ["framework/GpuClock.h"], + platforms = get_platforms(), + visibility = ["PUBLIC"], + ) + + runtime.cxx_test( + name = "gpu_clock_test", + srcs = ["framework/test/gpu_clock_test.cpp"], + platforms = get_platforms(), + deps = [":gpu_clock", "//third-party/googletest:gtest_main"], + ) + + runtime.cxx_library( + name = "framework", + srcs = [ + "framework/CaseCheckRegistry.cpp", + "framework/OperatorBenchmark.cpp", + "framework/ShaderBenchmark.cpp", + "framework/TensorUtils.cpp", + "framework/Timestamps.cpp", + "framework/Timing.cpp", + "framework/Validation.cpp", + "framework/WorkgroupUtils.cpp", + ], + exported_headers = [ + "framework/CaseCheckRegistry.h", + "framework/OperatorBenchmark.h", + "framework/ShaderBenchmark.h", + "framework/TensorUtils.h", + "framework/Timestamps.h", + "framework/Timing.h", + "framework/Validation.h", + "framework/WorkgroupUtils.h", + ], + platforms = get_platforms(), + exported_deps = [ + ":config", + ":gpu_clock", + ":report", + ":reference", + ":routes", + "//executorch/backends/vulkan:vulkan_graph_runtime", + ], + visibility = ["PUBLIC"], + ) + + runtime.cxx_test( + name = "data_initializer_test", + srcs = ["framework/test/data_initializer_test.cpp"], + platforms = get_platforms(), + deps = [":config", "//third-party/googletest:gtest_main"], + ) + + runtime.cxx_test( + name = "config_test", + srcs = ["framework/test/config_test.cpp"], + platforms = get_platforms(), + deps = [ + ":config", + "//third-party/googletest:gtest_main", + ], + ) + + runtime.cxx_test( + name = "validation_test", + srcs = ["framework/test/validation_test.cpp"], + platforms = get_platforms(), + deps = [ + "//executorch/backends/vulkan/test/api_abstractions:test_utils", + ":framework", + "//third-party/googletest:gtest_main", + ], + ) + + runtime.cxx_library( + name = "case_checks", + srcs = ["operators/convolution/q8ta/Q8taValidationChecks.cpp"], + exported_headers = ["operators/convolution/q8ta/Q8taValidationChecks.h"], + platforms = get_platforms(), + exported_deps = [":framework"], + # @lint-ignore BUCKLINT: Avoid `link_whole=True` + link_whole = True, + visibility = ["PUBLIC"], + ) + + runtime.cxx_test( + name = "case_check_registry_test", + srcs = ["framework/test/case_check_registry_test.cpp"], + platforms = get_platforms(), + deps = [ + "//executorch/backends/vulkan/test/api_abstractions:test_utils",":framework", "//third-party/googletest:gtest_main"], + ) + + runtime.cxx_test( + name = "migration_reference_test", + srcs = ["operators/MigrationReferenceTest.cpp"], + platforms = get_platforms(), + deps = [":reference", "//third-party/googletest:gtest_main"], + ) + + runtime.cxx_test( + name = "q8_benchmark_reference_test", + srcs = ["operators/quantize/q8ta/test/q8_benchmark_reference_test.cpp"], + platforms = get_platforms(), + deps = [":reference", "//third-party/googletest:gtest_main"], + ) + + runtime.cxx_test( + name = "q8ta_native_reference_test", + srcs = ["operators/convolution/q8ta/test/q8ta_native_reference_test.cpp"], + platforms = get_platforms(), + deps = [":reference", "//third-party/googletest:gtest_main"], + ) + + runtime.cxx_binary( + name = "etvk_ops_bench", + srcs = ["etvk_ops_bench.cpp"], + platforms = get_platforms(), + define_static_target = False, + deps = [ + ":framework", + ":case_checks", + ], + external_deps = ["gflags"], + )